hub v0.131.0: the tunnel status is true (R-841: three states, tunnel_down after two not_running reports, unknown never alarms); OS updates per layer (guest/host separate approved sets, host candidate leaves out kernel/boot/firmware, host_release in the box block), the fleet view and four hourly operator alarms (11 §8 steps 3+4); red-proofs in audits/os-host-lane-2026-10-04
gates / gates (push) Successful in 30s
gates / gates (push) Successful in 30s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
# agent v0.141.0 tunnel-probe red-proofs (each mutation compiles; result line printed)
|
||||
== mutation: container-state-only
|
||||
cloudflared_test.go:32: running but not connected: got "running" (container running but the tunnel is NOT connected (cloudflared /ready fails)), want "not_running" (…NOT connected…)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.009s
|
||||
== mutation: unknown-as-down
|
||||
cloudflared_test.go:32: guest not running: got "not_running" (could not ask the guest: CT 9201 not running), want "unknown" (…could not ask…)
|
||||
cloudflared_test.go:32: sudo refused: got "not_running" (could not ask the guest: sudo: a password is required), want "unknown" (…could not ask…)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.009s
|
||||
ok gitea.dooplex.hu/admin/felhom-agent/internal/hub (cached)
|
||||
@@ -0,0 +1,3 @@
|
||||
== RP (controller): the healthcheck block removed from the cloudflared template
|
||||
infra_tunnel_test.go:300: tunnel=false: compose lacks "test: [\"CMD\", \"cloudflared\", \"tunnel\", \"--metrics\", \"localhost:20241\", \"ready\"]":
|
||||
FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.007s
|
||||
@@ -0,0 +1,17 @@
|
||||
# hub v0.131.0 tunnel-alarm red-proofs, 2026-10-04 (each mutation compiles; the package result line is printed)
|
||||
== mutation: one-report
|
||||
--- FAIL: TestTunnelAlarm_Sequences/a_single_bad_report_does_not_alarm (0.02s)
|
||||
tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0
|
||||
--- FAIL: TestTunnelAlarm_Sequences/unknown_never_alarms_and_breaks_the_run (0.03s)
|
||||
tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0
|
||||
== mutation: unknown-counts-as-down
|
||||
--- FAIL: TestTunnelAlarm_Sequences/unknown_never_alarms_and_breaks_the_run (0.03s)
|
||||
tunnel_test.go:47: tunnel_down=1 tunnel_recovered=0, want 0 / 0
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.164s
|
||||
== mutation: no-alarm
|
||||
--- FAIL: TestTunnelAlarm_Sequences/two_in_a_row_alarm_once (0.03s)
|
||||
tunnel_test.go:47: tunnel_down=0 tunnel_recovered=0, want 1 / 0
|
||||
--- FAIL: TestTunnelAlarm_Sequences/recovery_after_an_alarm (0.04s)
|
||||
tunnel_test.go:47: tunnel_down=0 tunnel_recovered=1, want 1 / 1
|
||||
== reverted
|
||||
ok gitea.dooplex.hu/admin/felhom-hub/internal/api 0.168s
|
||||
@@ -0,0 +1,22 @@
|
||||
# agent v0.141.0 leg red-proofs (each mutation compiles; result line printed)
|
||||
== mutation: byo-gets-host
|
||||
--- FAIL: TestBYO_NoHostPlan (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.440s
|
||||
== mutation: host-after-failed-guest
|
||||
--- FAIL: TestGuestFailure_SkipsTheHost (0.00s)
|
||||
--- FAIL: TestHealth_FailsAfterTheWait (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.420s
|
||||
== mutation: host-ignores-services
|
||||
--- FAIL: TestHostHealthVerdict (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.441s
|
||||
== mutation: host-ignores-guest
|
||||
--- FAIL: TestHostHealthVerdict (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.410s
|
||||
== mutation: host-ignores-tunnel
|
||||
--- FAIL: TestHost_TunnelDownFailsTheHostStep (0.00s)
|
||||
--- FAIL: TestHostHealthVerdict (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.427s
|
||||
== mutation: host-uses-guest-release
|
||||
--- FAIL: TestRing1_EachLayerItsOwnRelease (0.00s)
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.461s
|
||||
ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate (cached)
|
||||
@@ -0,0 +1,20 @@
|
||||
# felhom-os-apply (agent v0.141.0) red-proof, 2026-10-04: each mutation is applied to a COPY, syntax-checked, and the suite run against it.
|
||||
R1 (7 raise(s) removed): compiles=True suite rc=1; failing: test_R1_not_owned_by_the_agent, test_R1_path_outside_the_plan_dir, test_R1_symlink; expected test failed: YES
|
||||
R2 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R2_non_debian_origin_in_the_plan, test_R2_non_debian_origin_in_the_simulation; expected test failed: YES
|
||||
R3 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R3_slow_lane; expected test failed: YES
|
||||
R4 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R4_removal; expected test failed: YES
|
||||
R5 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R5_downgrade_exact; expected test failed: YES
|
||||
R6 (4 raise(s) removed): compiles=True suite rc=1; failing: test_R6_allow_new_is_slow_lane, test_R6_new_package, test_R6_unlisted_package; expected test failed: YES
|
||||
R7 (7 raise(s) removed): compiles=True suite rc=1; failing: test_R7_not_downloadable_and_no_snapshot, test_snapshot_does_not_have_it_either; expected test failed: YES
|
||||
R8 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R8_free_space; expected test failed: YES
|
||||
R9 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R9_apt_lock_held, test_R9_guest_locked_by_a_backup; expected test failed: YES
|
||||
R10 (4 raise(s) removed): compiles=True suite rc=1; failing: test_R10_bind_only_in_a_snapshot_section, test_R10_not_running, test_R10_not_the_boxs_own_guest, test_R10_reserved_vmid; expected test failed: YES
|
||||
R11 (12 raise(s) removed): compiles=True suite rc=1; failing: test_R11_bad_name, test_R11_bad_version_string, test_R11_duplicate; expected test failed: YES
|
||||
R12 (5 raise(s) removed): compiles=True suite rc=1; failing: test_R12_host_on_a_byo_box, test_R12_host_record_not_root_owned, test_R12_host_without_an_install_record, test_R12_unknown_layer; expected test failed: YES
|
||||
R13 (1 raise(s) removed): compiles=True suite rc=1; failing: test_R13_repair_does_not_fix_it; expected test failed: YES
|
||||
R14 (2 raise(s) removed): compiles=True suite rc=1; failing: test_R14_kernel_package_in_a_host_plan, test_R14_kernel_package_pulled_by_the_simulation; expected test failed: YES
|
||||
host layer runs in the guest instead: compiles=True suite rc=1; failing: test_host_runs_on_the_host_not_in_the_guest; expected test failed: YES
|
||||
pending-fast takes every pending upgrade: compiles=True suite rc=1; failing: test_pending_fast_skips_proxmox_docker_and_kernel; expected test failed: YES
|
||||
version checks in the target (pct exec per package): compiles=True suite rc=1; failing: test_no_per_package_guest_calls; expected test failed: YES
|
||||
lxc-start does not mean reboot: compiles=True suite rc=0; failing: NONE; expected test failed: NO
|
||||
lxc-start does not mean reboot (after adding test_reboot_needed_for_lxc_start_alone): suite rc=1; failing: test_reboot_needed_for_lxc_start_alone
|
||||
@@ -0,0 +1,8 @@
|
||||
# Agent red-proof: the host_release wire tag (contract with the hub golden), 2026-10-04T11:04:25Z
|
||||
# mutation: report.go json:"host_release" -> json:"hostrelease"; restored after
|
||||
=== RUN TestOSUpdateGolden_Decodes
|
||||
osupdate_contract_test.go:34: host_release = <nil>
|
||||
--- FAIL: TestOSUpdateGolden_Decodes (0.00s)
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.007s
|
||||
FAIL
|
||||
@@ -0,0 +1,30 @@
|
||||
# Hub red-proofs, Parts B+C (hub v0.131.0), 2026-10-04T11:03:52Z
|
||||
# script: each mutation applied, the named test run with -v, file restored.
|
||||
[CAUGHT] host candidate leaves out kernel/boot/firmware: mutated internal/osupdates/service.go; TestCandidate_HostLeavesOutKernelBootFirmware ran=True failed=True rc=1
|
||||
alarms_test.go:27: host candidate = map[firmware-realtek:{Name:firmware-realtek Version:20250410-2 Origin:Debian} grub-efi-amd64:{Name:grub-efi-amd64 Version:2.12-9 Origin:Debian} intel-microcode:{Name:intel-microcode Ve
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.034s
|
||||
[CAUGHT] host nights counted per layer: mutated internal/osupdates/service.go; TestLayers_SeparateSets ran=True failed=True rc=1
|
||||
alarms_test.go:60: host approved on a GUEST night: &{ID:os-host-20261005-130000 Layer:host Fingerprint:535006df2f78491d ApprovedAt:2026-10-05 13:00:00 +0000 UTC ApprovedBy:auto PackagesJSON:[{"name":"libssl3t64","version
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.042s
|
||||
[CAUGHT] alarm 1 stale threshold: mutated internal/osupdates/service.go; TestAlarm_StaleLeg ran=True failed=True rc=1
|
||||
alarms_test.go:151: no alarm at 8 days: []
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.041s
|
||||
[CAUGHT] alarm 1 switch OFF not watched: mutated internal/osupdates/service.go; TestAlarm_StaleLeg ran=True failed=True rc=1
|
||||
alarms_test.go:168: alarmed with the switch OFF
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.054s
|
||||
[CAUGHT] alarm 2 reboot-needed counts from the FIRST report: mutated internal/osupdates/service.go; TestAlarm_RebootNeeded ran=True failed=True rc=1
|
||||
alarms_test.go:211: reboot-needed since = 2026-10-17 12:00:00 +0000 UTC
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.045s
|
||||
[CAUGHT] alarm 3 counts only fast-lane pending: mutated internal/osupdates/service.go; TestAlarm_Ring0Stalled ran=True failed=True rc=1
|
||||
alarms_test.go:235: a pending KERNEL counted as a stalled fast lane
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.040s
|
||||
[CAUGHT] alarm 4 ring 0 never alarms: mutated internal/osupdates/service.go; TestAlarm_NotCovered ran=True failed=True rc=1
|
||||
alarms_test.go:278: not-covered alarms = ["OS updates (guest): cust1 has had 1 fast-lane package(s) no approved release covers since 2026-10-04: libsomething-hw. Ring 0 does not run them (other hardware?) — add matching
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.060s
|
||||
[CAUGHT] alarm events are operator-only: mutated internal/notify/dispatcher.go; TestOSUpdateEvents_OperatorOnlyExceptApplied ran=True failed=True rc=1
|
||||
os_alarms_operator_test.go:17: os_ring0_stalled is not operator-only
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/notify 0.009s
|
||||
[CAUGHT] host_release served to ring 1: mutated internal/osupdates/service.go; TestOSUpdate_DesiredBlockMatchesTheGolden ran=True failed=True rc=1
|
||||
os_updates_test.go:59: served os_update diverged from the golden:
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.055s
|
||||
after restore: ok gitea.dooplex.hu/admin/felhom-hub/internal/osupdates 0.544s | ok gitea.dooplex.hu/admin/felhom-hub/internal/notify 1.169s | ok gitea.dooplex.hu/admin/felhom-hub/internal/api 4.690s rc= 0
|
||||
@@ -0,0 +1,44 @@
|
||||
import subprocess, sys, shutil
|
||||
M = [
|
||||
("host candidate leaves out kernel/boot/firmware", "internal/osupdates/service.go",
|
||||
"if layer == LayerHost && hostSlowRE.MatchString(p.Name) {\n\t\t\t\tcontinue", "if false && hostSlowRE.MatchString(p.Name) {\n\t\t\t\tcontinue",
|
||||
"./internal/osupdates/", "TestCandidate_HostLeavesOutKernelBootFirmware"),
|
||||
("host nights counted per layer", "internal/osupdates/service.go",
|
||||
"reps, err := s.Store.OSReportsSince(h, layer, first)", "reps, err := s.Store.OSReportsSince(h, LayerGuest, first)",
|
||||
"./internal/osupdates/", "TestLayers_SeparateSets"),
|
||||
("alarm 1 stale threshold", "internal/osupdates/service.go",
|
||||
"now.Sub(ref) >= stale", "now.Sub(ref) >= 10*stale", "./internal/osupdates/", "TestAlarm_StaleLeg"),
|
||||
("alarm 1 switch OFF not watched", "internal/osupdates/service.go",
|
||||
"holds := st.Enabled && canRun", "holds := canRun", "./internal/osupdates/", "TestAlarm_StaleLeg"),
|
||||
("alarm 2 reboot-needed counts from the FIRST report", "internal/osupdates/service.go",
|
||||
"\t\tsince = rep.ReceivedAt\n", "\t\tif since.IsZero() {\n\t\t\tsince = rep.ReceivedAt\n\t\t}\n", "./internal/osupdates/", "TestAlarm_RebootNeeded"),
|
||||
("alarm 3 counts only fast-lane pending", "internal/osupdates/service.go",
|
||||
"pendingAny += len(fastPending(layer, r.Pending))", "pendingAny += len(r.Pending)", "./internal/osupdates/", "TestAlarm_Ring0Stalled"),
|
||||
("alarm 4 ring 0 never alarms", "internal/osupdates/service.go",
|
||||
"\tif ring == 0 {\n\t\treturn nil\n\t}\n", "", "./internal/osupdates/", "TestAlarm_NotCovered"),
|
||||
("alarm events are operator-only", "internal/notify/dispatcher.go",
|
||||
'\t"os_ring0_stalled": true,\n', "", "./internal/notify/", "TestOSUpdateEvents_OperatorOnlyExceptApplied"),
|
||||
("host_release served to ring 1", "internal/osupdates/service.go",
|
||||
"b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost)", "b.Release = s.releaseBlock(LayerGuest)",
|
||||
"./internal/api/", "TestOSUpdate_DesiredBlockMatchesTheGolden"),
|
||||
]
|
||||
ok = True
|
||||
for name, f, old, new, pkg, test in M:
|
||||
src = open(f).read()
|
||||
assert src.count(old) == 1, (name, src.count(old))
|
||||
shutil.copy(f, f + ".bak")
|
||||
open(f, "w").write(src.replace(old, new))
|
||||
r = subprocess.run(["go", "test", pkg, "-run", "^" + test + "$", "-v", "-count=1"], capture_output=True, text=True)
|
||||
shutil.move(f + ".bak", f)
|
||||
out = r.stdout + r.stderr
|
||||
ran = ("=== RUN " + test) in out
|
||||
failed = ("--- FAIL: " + test) in out
|
||||
verdict = "CAUGHT" if (ran and failed and r.returncode != 0) else "NOT CAUGHT"
|
||||
if verdict != "CAUGHT": ok = False
|
||||
print(f"[{verdict}] {name}: mutated {f}; {test} ran={ran} failed={failed} rc={r.returncode}")
|
||||
for l in out.splitlines():
|
||||
if "_test.go:" in l or l.startswith("FAIL\t") or "build failed" in l or l.startswith("#"):
|
||||
print(" " + l.strip()[:220])
|
||||
r = subprocess.run(["go", "test", "./internal/osupdates/", "./internal/notify/", "./internal/api/", "-count=1"], capture_output=True, text=True)
|
||||
print("after restore:", r.stdout.strip().replace("\n", " | "), "rc=", r.returncode)
|
||||
sys.exit(0 if ok and r.returncode == 0 else 1)
|
||||
@@ -1,3 +1,32 @@
|
||||
## v0.131.0 — the tunnel status is true (R-841); OS updates: the host fast lane's own approved set, the fleet view and four alarms (`11` §8 steps 3+4) (2026-10-04)
|
||||
|
||||
**Needs agent v0.141.0** for the tunnel's three states and the host layer. An older agent still reports
|
||||
`cloudflared.status` (read as before) and never sends a host-layer report — nothing breaks.
|
||||
|
||||
- **R-841 — three tunnel states.** The agent now reports `running` / `not_running` / `unknown` plus a short `detail`
|
||||
(agent v0.141.0 reads the guest's cloudflared container and its readiness check). The host list and the host page
|
||||
show the state and the detail. **`unknown` never alarms.** A tunnel `not_running` in the newest TWO reports (more
|
||||
than one report cycle) raises `tunnel_down` (error, operator-only, mailed once per run); the first `running` after
|
||||
that raises `tunnel_recovered` (info). An `unknown` report breaks the run. Pinned by `api/tunnel_test.go`.
|
||||
- **Two layers, two approved sets.** OS reports and OS releases carry a `layer` (`guest` | `host`; old rows read as
|
||||
`guest`). The candidate, the 24 h + 1 night rule and "every ring-0 box" are applied PER LAYER: a guest night run
|
||||
does not count toward a host approval. Release ids are `os-guest-…` / `os-host-…`. The host candidate leaves out
|
||||
kernel, boot and firmware names (the host's slow lane; the same pattern as the wrapper's R14 refusal).
|
||||
Ring 1 receives `os_update.host_release` beside `release` — golden updated in BOTH repos.
|
||||
- **The fleet view** (`GET /os/fleet`, operator): one line per box — ring, switch, the tunnel, and per layer: the
|
||||
release, last outcome, last successful leg, pending, not-covered, restart-needed, reboot-needed-since (the first
|
||||
report that said so) and the wrapper's own seconds.
|
||||
- **Four alarms**, checked hourly, each operator-only, raised at most once a week and cleared when the condition goes
|
||||
away: `os_update_stale` (no successful OS leg for 7 days while the switch is ON; agents ≥ 0.140.0 only; the mail
|
||||
names the likely reason), `os_reboot_needed` (host reboot needed for 14 days), `os_ring0_stalled` (ring 0 approved
|
||||
nothing for 7 days while it has pending fast-lane updates — a pending kernel does not count), `os_not_covered`
|
||||
(a ring-1 box has fast-lane packages no release covers for 14 days). The numbers are configuration
|
||||
(`OS_ALARM_STALE_AFTER`, `OS_ALARM_REBOOT_AFTER`, `OS_ALARM_RING0_STALL_AFTER`, `OS_ALARM_NOT_COVERED_AFTER`),
|
||||
logged at start; recorded in `11` §8.3 as decided by CC unattended — operator may reverse.
|
||||
- **Host health mail** names the host undo runbook (`runbooks/os-updates-host-undo.md`), not the guest backup.
|
||||
- Red-proofs: `documentation/audits/os-host-lane-2026-10-04/partA/hub-redproofs.txt` (tunnel) and
|
||||
`partC/hub-redproofs.txt` (9 mutations, all caught).
|
||||
|
||||
## v0.130.0 — OS updates, guest fast lane: rings, the per-box switch, OS releases approved from ring 0 (`11` §8 step 2)
|
||||
|
||||
- **The OS release.** Ring-0 boxes (the demo boxes) report every OS-leg run to `POST /api/v1/hosts/{id}/os-report`
|
||||
|
||||
+40
-6
@@ -414,23 +414,57 @@ func main() {
|
||||
logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired)
|
||||
apiHandler.SetOSUpdateService(osSvc)
|
||||
webServer.SetOSUpdateAdmin(osSvc)
|
||||
// `11` §8.3: the four alarm thresholds are configuration (decided by CC unattended — operator may reverse).
|
||||
for _, a := range []struct {
|
||||
env string
|
||||
dst *time.Duration
|
||||
def time.Duration
|
||||
}{
|
||||
{"OS_ALARM_STALE_AFTER", &osSvc.StaleAfter, 7 * 24 * time.Hour},
|
||||
{"OS_ALARM_REBOOT_AFTER", &osSvc.RebootAfter, 14 * 24 * time.Hour},
|
||||
{"OS_ALARM_RING0_STALL_AFTER", &osSvc.Ring0StallAfter, 7 * 24 * time.Hour},
|
||||
{"OS_ALARM_NOT_COVERED_AFTER", &osSvc.NotCoveredAfter, 14 * 24 * time.Hour},
|
||||
} {
|
||||
*a.dst = a.def
|
||||
if v := os.Getenv(a.env); v != "" {
|
||||
if d, derr := time.ParseDuration(v); derr == nil && d > 0 {
|
||||
*a.dst = d
|
||||
} else {
|
||||
logger.Printf("[ERROR] %s=%q invalid — keeping %s", a.env, v, a.def)
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Printf("[INFO] osupdates: alarms = no OS leg %s, reboot needed %s, ring 0 stalled %s, not covered %s",
|
||||
osSvc.StaleAfter, osSvc.RebootAfter, osSvc.Ring0StallAfter, osSvc.NotCoveredAfter)
|
||||
go func() {
|
||||
tk := time.NewTicker(60 * time.Second)
|
||||
defer tk.Stop()
|
||||
last := ""
|
||||
last := map[string]string{}
|
||||
var lastAlarms time.Time
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tk.C:
|
||||
st, err := osSvc.Evaluate()
|
||||
sts, err := osSvc.Evaluate()
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] osupdates: evaluate: %v", err)
|
||||
continue
|
||||
}
|
||||
if msg := st.Fingerprint + "|" + st.Waiting; msg != last {
|
||||
last = msg
|
||||
logger.Printf("[INFO] osupdates: candidate %s (%d packages, first seen %s): %s", st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""])
|
||||
for _, st := range sts {
|
||||
if msg := st.Fingerprint + "|" + st.Waiting; msg != last[st.Layer] {
|
||||
last[st.Layer] = msg
|
||||
logger.Printf("[INFO] osupdates: %s candidate %s (%d packages, first seen %s): %s", st.Layer, st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""])
|
||||
}
|
||||
}
|
||||
if time.Since(lastAlarms) >= time.Hour {
|
||||
lastAlarms = time.Now()
|
||||
start := time.Now()
|
||||
sent, aerr := osSvc.Alarms()
|
||||
if aerr != nil {
|
||||
logger.Printf("[WARN] osupdates: alarms: %v", aerr)
|
||||
} else {
|
||||
logger.Printf("[INFO] osupdates: alarm check done in %s, %d alarm(s) sent %v", time.Since(start).Round(time.Millisecond), len(sent), sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -689,7 +689,8 @@ type hostReportPayload struct {
|
||||
RestoreTests []hostRestoreTest `json:"restore_tests"` // slice 6
|
||||
PBSSnapshots []hostPBSSnapshot `json:"pbs_snapshots"` // slice 6 Phase B
|
||||
Cloudflared struct {
|
||||
Status string `json:"status"`
|
||||
Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown (older: active | inactive | …)
|
||||
Detail string `json:"detail,omitempty"`
|
||||
} `json:"cloudflared"`
|
||||
// DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim,
|
||||
// ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half.
|
||||
@@ -861,6 +862,8 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
|
||||
for _, g := range rep.Guests {
|
||||
status := g.Status
|
||||
if status == "" {
|
||||
@@ -2835,3 +2838,39 @@ func (h *Handler) handleAssetFile(w http.ResponseWriter, r *http.Request, filena
|
||||
|
||||
h.assetsMgr.ServeFile(w, r, filename)
|
||||
}
|
||||
|
||||
// Tunnel alarm (R-841, hub v0.131.0). The agent (>= 0.141.0) reports the box's tunnel as running | not_running |
|
||||
// unknown, read from the cloudflared container's own readiness check. `not_running` in TWO reports in a row — more
|
||||
// than one report cycle — is an operator alarm (`tunnel_down`, error); the first `running` after that is
|
||||
// `tunnel_recovered`. `unknown` never alarms and breaks a run of not_running (R-96 rule 3: a probe that could not
|
||||
// ask is not evidence of down). Pinned by TestTunnelAlarm_*.
|
||||
const (
|
||||
eventTunnelDown = "tunnel_down"
|
||||
eventTunnelRecovered = "tunnel_recovered"
|
||||
)
|
||||
|
||||
func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
st, err := h.store.RecentCloudflaredStatuses(hostID, 3)
|
||||
if err != nil || len(st) < 2 {
|
||||
return
|
||||
}
|
||||
down := func(i int) bool { return i < len(st) && st[i] == "not_running" }
|
||||
var typ, sev, msg string
|
||||
switch {
|
||||
case down(0) && down(1) && !down(2):
|
||||
typ, sev = eventTunnelDown, "error"
|
||||
msg = fmt.Sprintf("The tunnel of %s is NOT running in two reports in a row (%s). The box is not reachable from outside its home.", hostID, detail)
|
||||
case st[0] == "running" && down(1) && down(2):
|
||||
typ, sev = eventTunnelRecovered, "info"
|
||||
msg = fmt.Sprintf("The tunnel of %s is running again.", hostID)
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s tunnel: %s (%s)", hostID, typ, detail)
|
||||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "status": status, "detail": detail})
|
||||
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,11 @@ func TestOSUpdate_DesiredBlockMatchesTheGolden(t *testing.T) {
|
||||
svc := &osupdates.Service{Store: st, ApproveAfter: 0, NightsRequired: 0, Logger: log.New(os.Stderr, "", 0)}
|
||||
h.SetOSUpdateService(svc)
|
||||
rel := `[{"name":"libc6","version":"2.41-12+deb13u4","origin":"Debian"},{"name":"openssl","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]`
|
||||
if err := st.SaveOSRelease(storeRelease("os-20261004-120000", rel)); err != nil {
|
||||
if err := st.SaveOSRelease(storeRelease("os-guest-20261004-120000", "guest", rel)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hrel := `[{"name":"libssl3t64","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]`
|
||||
if err := st.SaveOSRelease(storeRelease("os-host-20261004-120000", "host", hrel)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr := do(h, http.MethodGet, "/hosts/h1/desired-state", "HKEY1", "")
|
||||
@@ -66,18 +70,18 @@ func TestOSReport_SelfScoped(t *testing.T) {
|
||||
if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY2", body); rr.Code != http.StatusForbidden {
|
||||
t.Fatalf("cross-host report → %d, want 403", rr.Code)
|
||||
}
|
||||
if r, _ := st.LatestOSReport("h1"); r != nil {
|
||||
if r, _ := st.LatestOSReport("h1", "guest"); r != nil {
|
||||
t.Fatal("a refused report was stored")
|
||||
}
|
||||
if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY1", body); rr.Code != http.StatusOK {
|
||||
t.Fatalf("own report → %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if r, _ := st.LatestOSReport("h1"); r == nil || r.Outcome != "applied" {
|
||||
if r, _ := st.LatestOSReport("h1", "guest"); r == nil || r.Outcome != "applied" {
|
||||
t.Fatalf("report not stored: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func storeRelease(id, pkgs string) store.OSRelease {
|
||||
func storeRelease(id, layer, pkgs string) store.OSRelease {
|
||||
at, _ := time.Parse(time.RFC3339, "2026-10-04T12:00:00Z")
|
||||
return store.OSRelease{ID: id, Fingerprint: "fp", ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs}
|
||||
return store.OSRelease{ID: id, Layer: layer, Fingerprint: "fp-" + layer, ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,19 @@
|
||||
"ring": 1,
|
||||
"enabled": true,
|
||||
"release": {
|
||||
"id": "os-20261004-120000",
|
||||
"id": "os-guest-20261004-120000",
|
||||
"snapshot": "20261004T120000Z",
|
||||
"packages": [
|
||||
{"name": "libc6", "version": "2.41-12+deb13u4", "origin": "Debian"},
|
||||
{"name": "openssl", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"}
|
||||
]
|
||||
},
|
||||
"host_release": {
|
||||
"id": "os-host-20261004-120000",
|
||||
"snapshot": "20261004T120000Z",
|
||||
"packages": [
|
||||
{"name": "libssl3t64", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
func tunnelReport(status, detail string) string {
|
||||
return strings.Replace(validReportBody("h1"), `"cloudflared":{"status":"active"}`,
|
||||
`"cloudflared":{"status":"`+status+`","detail":"`+detail+`"}`, 1)
|
||||
}
|
||||
|
||||
// R-841: the tunnel alarm fires when the tunnel is not running in TWO reports in a row; running never alarms;
|
||||
// one bad report never alarms; `unknown` never alarms and breaks a run; recovery is announced once.
|
||||
// Red-proof: make checkTunnel alarm on ONE not_running (drop down(1)) and "a single bad report" fails.
|
||||
func TestTunnelAlarm_Sequences(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
seq []string
|
||||
down int
|
||||
recov int
|
||||
}{
|
||||
{"running never alarms", []string{"running", "running", "running"}, 0, 0},
|
||||
{"a single bad report does not alarm", []string{"running", "not_running", "running"}, 0, 0},
|
||||
{"two in a row alarm once", []string{"running", "not_running", "not_running", "not_running"}, 1, 0},
|
||||
{"unknown never alarms and breaks the run", []string{"not_running", "unknown", "not_running", "unknown"}, 0, 0},
|
||||
{"recovery after an alarm", []string{"not_running", "not_running", "running"}, 1, 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
|
||||
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
|
||||
for _, s := range c.seq {
|
||||
if rr := do(h, http.MethodPost, "/host-report", "HKEY", tunnelReport(s, "exited, exit code 1")); rr.Code != 200 {
|
||||
t.Fatalf("report: %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
since := time.Now().Add(-time.Hour)
|
||||
d, _ := st.GetEventsByType("c1", eventTunnelDown, since)
|
||||
r, _ := st.GetEventsByType("c1", eventTunnelRecovered, since)
|
||||
if len(d) != c.down || len(r) != c.recov {
|
||||
t.Fatalf("tunnel_down=%d tunnel_recovered=%d, want %d / %d", len(d), len(r), c.down, c.recov)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -687,6 +687,14 @@ var operatorOnlyEvents = map[string]bool{
|
||||
"os_release_approved": true,
|
||||
"os_release_approved_now": true,
|
||||
"os_update_settings_changed": true,
|
||||
// R-841 (hub v0.131.0): the tunnel alarm — a box fact the household can do nothing about from inside.
|
||||
"tunnel_down": true,
|
||||
"tunnel_recovered": true,
|
||||
// `11` §8.3 (hub v0.131.0): the four OS-update alarms — fleet facts only the operator can act on.
|
||||
"os_update_stale": true,
|
||||
"os_reboot_needed": true,
|
||||
"os_ring0_stalled": true,
|
||||
"os_not_covered": true,
|
||||
// R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow
|
||||
// blobs. A customer can take no action on it — the remedy is the operator's inspection of the
|
||||
// off-site tier — and the text is operator-grade English naming host ids and retained-blob
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
|
||||
)
|
||||
|
||||
// Every OS-update event except the household's own line is operator-only (`11` §5.7, §8.3). An operator alarm left
|
||||
// off this list reaches the customer as raw operator English. Red-proof: delete one alarm line from
|
||||
// operatorOnlyEvents and this fails naming it.
|
||||
func TestOSUpdateEvents_OperatorOnlyExceptApplied(t *testing.T) {
|
||||
for _, e := range []string{osupdates.EventFailed, osupdates.EventHealthFailed, osupdates.EventReleaseApprove,
|
||||
osupdates.EventApprovedNow, osupdates.EventSettings, osupdates.EventStale, osupdates.EventRebootNeeded,
|
||||
osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered"} {
|
||||
if !operatorOnlyEvents[e] {
|
||||
t.Errorf("%s is not operator-only", e)
|
||||
}
|
||||
}
|
||||
if operatorOnlyEvents[osupdates.EventApplied] {
|
||||
t.Errorf("%s is the household's line and must NOT be operator-only", osupdates.EventApplied)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package osupdates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// ---------- the host layer (`11` §8 step 3) ----------
|
||||
|
||||
// The host release never carries a kernel, boot or firmware package (the host's slow lane, C3): the wrapper would
|
||||
// refuse the whole plan (R14). Red-proof: drop the hostSlowRE check in candidate() and this fails.
|
||||
func TestCandidate_HostLeavesOutKernelBootFirmware(t *testing.T) {
|
||||
f := newFix(t)
|
||||
set := []Package{pk("libssl3t64", "3.5.7-1~deb13u3"), pk("proxmox-kernel-6.14.11-4-pve-signed", "6.14.11-4"),
|
||||
pk("grub-efi-amd64", "2.12-9"), pk("firmware-realtek", "20250410-2"), pk("intel-microcode", "3.20250512.1"),
|
||||
pk("shim-signed", "1.46"), pk("pve-firmware", "3.16-3")}
|
||||
f.reportL(t, "hp", LayerHost, "night", true, set...)
|
||||
f.reportL(t, "n100", LayerHost, "night", true, set...)
|
||||
cand, err := f.s.candidate(LayerHost, []string{"hp", "n100"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cand) != 1 || cand["libssl3t64"].Version != "3.5.7-1~deb13u3" {
|
||||
t.Fatalf("host candidate = %+v, want only libssl3t64", cand)
|
||||
}
|
||||
// The guest layer has no such rule: a guest has no kernel of its own, nothing to leave out.
|
||||
f.reportL(t, "hp", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1"))
|
||||
f.reportL(t, "n100", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1"))
|
||||
if g, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"}); len(g) != 1 {
|
||||
t.Fatalf("guest candidate = %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// Host and guest are SEPARATE approved sets: a guest night run does not count toward a host approval, and each layer
|
||||
// gets its own release id. Red-proof: count nights across layers (OSReportsSince without layer) and this fails.
|
||||
func TestLayers_SeparateSets(t *testing.T) {
|
||||
f := newFix(t)
|
||||
g := []Package{pk("libc6", "2.41-12+deb13u4")}
|
||||
h := []Package{pk("libssl3t64", "3.5.7-1~deb13u3")}
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.reportL(t, b, LayerGuest, "debug", true, g...)
|
||||
f.reportL(t, b, LayerHost, "debug", true, h...)
|
||||
}
|
||||
f.s.Evaluate()
|
||||
f.now = f.now.Add(25 * time.Hour)
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.reportL(t, b, LayerGuest, "night", true, g...) // a GUEST night only
|
||||
}
|
||||
sts, err := f.s.Evaluate()
|
||||
if err != nil || len(sts) != 2 {
|
||||
t.Fatalf("%v %+v", err, sts)
|
||||
}
|
||||
if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil || !strings.HasPrefix(gr.ID, "os-guest-") {
|
||||
t.Fatalf("guest release = %+v", gr)
|
||||
}
|
||||
if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil {
|
||||
t.Fatalf("host approved on a GUEST night: %+v", hr)
|
||||
}
|
||||
if !strings.Contains(sts[1].Waiting, "night run") || sts[1].Layer != LayerHost {
|
||||
t.Fatalf("host status = %+v", sts[1])
|
||||
}
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.reportL(t, b, LayerHost, "night", true, h...)
|
||||
}
|
||||
f.s.Evaluate()
|
||||
hr, _ := f.s.Store.LatestOSRelease(LayerHost)
|
||||
if hr == nil || !strings.HasPrefix(hr.ID, "os-host-") {
|
||||
t.Fatalf("host release = %+v", hr)
|
||||
}
|
||||
b := f.s.DesiredBlock("cust1")
|
||||
if b.Release == nil || b.HostRelease == nil || b.HostRelease.Packages[0].Name != "libssl3t64" || b.Release.Packages[0].Name != "libc6" {
|
||||
t.Fatalf("ring-1 block = %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
// An unhealthy HOST run blocks only the host approval. The undo text points to the host runbook, not the guest backup.
|
||||
func TestHostUnhealthy_BlocksHostOnly(t *testing.T) {
|
||||
f := newFix(t)
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1"))
|
||||
f.reportL(t, b, LayerHost, "night", true, pk("libssl3t64", "1"))
|
||||
}
|
||||
f.s.Evaluate()
|
||||
f.now = f.now.Add(25 * time.Hour)
|
||||
var msgs []string
|
||||
f.s.Emit = func(_, typ, _, msg, _, _ string) { f.events = append(f.events, typ); msgs = append(msgs, msg) }
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1"))
|
||||
}
|
||||
f.reportL(t, "hp", LayerHost, "night", false, pk("libssl3t64", "1"))
|
||||
f.reportL(t, "n100", LayerHost, "night", true, pk("libssl3t64", "1"))
|
||||
f.s.Evaluate()
|
||||
if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil {
|
||||
t.Fatal("an unhealthy HOST run blocked the GUEST approval")
|
||||
}
|
||||
if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil {
|
||||
t.Fatalf("host approved over an unhealthy host run: %+v", hr)
|
||||
}
|
||||
if !strings.Contains(strings.Join(msgs, "\n"), "os-updates-host-undo.md") {
|
||||
t.Fatalf("host health mail does not name the host undo runbook: %q", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- the four alarms (`11` §8.3) ----------
|
||||
|
||||
func (f *fix) count(typ string) int {
|
||||
n := 0
|
||||
for _, e := range f.events {
|
||||
if e == typ {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (f *fix) setAgent(t *testing.T, host, ver string) {
|
||||
t.Helper()
|
||||
if err := f.s.Store.SaveHostReport(host, "c-"+host, []byte(`{}`), store.HostReportDenorm{AgentVersion: ver}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fix) ingest(t *testing.T, host string, r Report) {
|
||||
t.Helper()
|
||||
if r.RunID == "" {
|
||||
r.RunID = host + f.now.String()
|
||||
}
|
||||
if err := f.s.Ingest(host, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Alarm 1: no completed OS leg for 7 days while the switch is ON, naming the likely reason; once a week at most;
|
||||
// cleared by a good leg. Red-proof: compare against a fixed 70 days (or drop the Enabled check) and a sub-step fails.
|
||||
func TestAlarm_StaleLeg(t *testing.T) {
|
||||
f := newFix(t)
|
||||
f.s.Now = func() time.Time { return f.now }
|
||||
f.now = time.Now().UTC()
|
||||
f.setAgent(t, "cust1", "0.141.0")
|
||||
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
||||
f.now = f.now.Add(6 * 24 * time.Hour)
|
||||
if sent, _ := f.s.Alarms(); f.count(EventStale) != 0 {
|
||||
t.Fatalf("alarm at 6 days: %v", sent)
|
||||
}
|
||||
f.now = f.now.Add(2 * 24 * time.Hour)
|
||||
f.s.Alarms()
|
||||
if f.count(EventStale) != 1 {
|
||||
t.Fatalf("no alarm at 8 days: %v", f.events)
|
||||
}
|
||||
f.now = f.now.Add(time.Hour)
|
||||
f.s.Alarms()
|
||||
if f.count(EventStale) != 1 {
|
||||
t.Fatal("re-alarmed within the week")
|
||||
}
|
||||
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true})
|
||||
f.s.Alarms()
|
||||
if raised := f.s.Store.OSAlarmRaised("stale:cust1"); !raised.IsZero() {
|
||||
t.Fatal("a good leg did not clear the alarm")
|
||||
}
|
||||
// The switch OFF: the leg is not expected, no alarm.
|
||||
_ = f.s.Store.SetOSEnabled("cust1", false)
|
||||
f.now = f.now.Add(30 * 24 * time.Hour)
|
||||
f.s.Alarms()
|
||||
if f.count(EventStale) != 1 {
|
||||
t.Fatal("alarmed with the switch OFF")
|
||||
}
|
||||
}
|
||||
|
||||
// The stale alarm's reason: a failed leg is named; a box that runs an agent too old for the leg is not watched.
|
||||
func TestAlarm_StaleNamesTheReason(t *testing.T) {
|
||||
f := newFix(t)
|
||||
var msgs []string
|
||||
f.s.Emit = func(_, typ, _, msg, _, _ string) {
|
||||
if typ == EventStale {
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
}
|
||||
// The leg reports are dated 8 days back; the host reports (real clock) are fresh — the box IS reporting.
|
||||
f.now = time.Now().UTC().Add(-8*24*time.Hour - time.Hour)
|
||||
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
||||
f.now = f.now.Add(time.Hour)
|
||||
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "refused", HealthReason: "R6 dpkg lock"})
|
||||
f.setAgent(t, "cust1", "0.141.0")
|
||||
f.setAgent(t, "hp", "0.139.0") // too old to run the leg: never watched
|
||||
f.now = time.Now().UTC()
|
||||
f.s.Alarms()
|
||||
if len(msgs) != 1 || !strings.Contains(msgs[0], "ended refused") || !strings.Contains(msgs[0], "Likely reason") {
|
||||
t.Fatalf("stale alarms = %q", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
// Alarm 2: reboot needed for more than 14 days (host layer), from the FIRST report that said so.
|
||||
// Red-proof: take `since` from the newest report instead of the oldest in the run and the 15-day step stays silent.
|
||||
func TestAlarm_RebootNeeded(t *testing.T) {
|
||||
f := newFix(t)
|
||||
up := []Package{pk("libc6", "2")}
|
||||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true})
|
||||
for d := 1; d <= 13; d++ {
|
||||
f.now = f.now.Add(24 * time.Hour)
|
||||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
||||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true})
|
||||
}
|
||||
f.s.Alarms()
|
||||
if f.count(EventRebootNeeded) != 0 {
|
||||
t.Fatal("alarm at 13 days")
|
||||
}
|
||||
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() || f.now.Sub(l.RebootNeededSince) != 13*24*time.Hour {
|
||||
t.Fatalf("reboot-needed since = %v", l.RebootNeededSince)
|
||||
}
|
||||
f.now = f.now.Add(2 * 24 * time.Hour)
|
||||
f.s.Alarms()
|
||||
if f.count(EventRebootNeeded) != 1 {
|
||||
t.Fatalf("no alarm at 15 days: %v", f.events)
|
||||
}
|
||||
// A guest-layer reboot flag never feeds the host alarm.
|
||||
if l := f.s.layerLine("hp", LayerGuest, 0); !l.RebootNeededSince.IsZero() {
|
||||
t.Fatal("guest line shows a reboot need")
|
||||
}
|
||||
}
|
||||
|
||||
// Alarm 3: ring 0 approved nothing for 7 days while it has pending fast-lane updates. Pending SLOW-lane packages
|
||||
// (a new kernel) do not count. Red-proof: count all pending (not fastPending) and the kernel-only step alarms.
|
||||
func TestAlarm_Ring0Stalled(t *testing.T) {
|
||||
f := newFix(t)
|
||||
kernel := []PendingPkg{{Name: "proxmox-kernel-6.17", From: "", To: "6.17.1", Origin: []string{"Debian"}}}
|
||||
for _, b := range []string{"hp", "n100"} {
|
||||
f.ingest(t, b, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: kernel})
|
||||
}
|
||||
f.now = f.now.Add(8 * 24 * time.Hour)
|
||||
f.s.Alarms()
|
||||
if f.count(EventRing0Stalled) != 0 {
|
||||
t.Fatal("a pending KERNEL counted as a stalled fast lane")
|
||||
}
|
||||
fast := []PendingPkg{{Name: "libssl3t64", From: "1", To: "2", Origin: []string{"Debian-Security"}}}
|
||||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "failed", Healthy: false, Pending: fast})
|
||||
f.s.Alarms()
|
||||
if f.count(EventRing0Stalled) != 1 {
|
||||
t.Fatalf("no stall alarm: %v", f.events)
|
||||
}
|
||||
// A release clears it.
|
||||
_ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-host-x", Layer: LayerHost, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", PackagesJSON: "[]"})
|
||||
f.s.Alarms()
|
||||
if !f.s.Store.OSAlarmRaised("ring0stall:host").IsZero() {
|
||||
t.Fatal("a new release did not clear the stall alarm")
|
||||
}
|
||||
}
|
||||
|
||||
// Alarm 4: a ring-1 box with fast-lane packages no release covers for more than 14 days. Ring 0 never alarms.
|
||||
// Red-proof: drop the ring==0 early return in notCoveredFast and the ring-0 step alarms.
|
||||
func TestAlarm_NotCovered(t *testing.T) {
|
||||
f := newFix(t)
|
||||
odd := []PendingPkg{{Name: "libsomething-hw", From: "1", To: "2", Origin: []string{"Debian"}}}
|
||||
_ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-guest-x", Layer: LayerGuest, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto",
|
||||
PackagesJSON: `[{"name":"libc6","version":"2","origin":"Debian"}]`})
|
||||
for d := 0; d <= 15; d++ {
|
||||
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd})
|
||||
f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd})
|
||||
if d == 13 {
|
||||
f.s.Alarms()
|
||||
if f.count(EventNotCovered) != 0 {
|
||||
t.Fatal("alarm at 13 days")
|
||||
}
|
||||
}
|
||||
f.now = f.now.Add(24 * time.Hour)
|
||||
}
|
||||
var msgs []string
|
||||
f.s.Emit = func(_, typ, _, msg, _, _ string) {
|
||||
f.events = append(f.events, typ)
|
||||
if typ == EventNotCovered {
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
}
|
||||
f.s.Alarms()
|
||||
if len(msgs) != 1 || !strings.Contains(msgs[0], "cust1") || !strings.Contains(msgs[0], "libsomething-hw") {
|
||||
t.Fatalf("not-covered alarms = %q (a ring-0 box must never raise one)", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
// The fleet view: one line per box, both layers, the tunnel.
|
||||
func TestFleet_OneLinePerBoxBothLayers(t *testing.T) {
|
||||
f := newFix(t)
|
||||
if err := f.s.Store.SaveHostReport("hp", "c-hp", []byte(`{}`), store.HostReportDenorm{AgentVersion: "0.141.0", CloudflaredStatus: "running"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: []Package{pk("a", "1")}, PassSeconds: 8.5})
|
||||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true,
|
||||
Pending: []PendingPkg{{Name: "proxmox-kernel-6.17", Origin: []string{"Debian"}}}})
|
||||
lines, err := f.s.Fleet()
|
||||
if err != nil || len(lines) != 3 {
|
||||
t.Fatalf("%v %d lines", err, len(lines))
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.HostID != "hp" {
|
||||
continue
|
||||
}
|
||||
if l.Tunnel != "running" || l.Guest.LastOutcome != "applied" || l.Guest.WrapperPassSeconds != 8.5 ||
|
||||
l.Host.LastOutcome != "nothing" || l.Host.Pending != 1 || l.Host.LastSuccessfulLeg.IsZero() {
|
||||
t.Fatalf("hp line = %+v", l)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("no hp line")
|
||||
}
|
||||
+422
-107
@@ -1,14 +1,19 @@
|
||||
// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0).
|
||||
// Package osupdates is the hub half of the OS-update lanes (`11-os-updates.md` §5.3, §8 steps 2–4).
|
||||
//
|
||||
// Two LAYERS, each with its own approved set (hub v0.131.0): the customer guest's Debian packages and the Proxmox
|
||||
// host's Debian packages. A version approved for one layer is NOT approved for the other by that fact alone.
|
||||
//
|
||||
// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed
|
||||
// set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that
|
||||
// package agree on. A candidate is APPROVED when, since it was first seen:
|
||||
// - every ring-0 box runs it (its newest report matches the candidate for every package it has),
|
||||
// - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and
|
||||
// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs.
|
||||
// set per layer (C9). Per layer, the CANDIDATE is every Debian-origin package=version that all ring-0 boxes having it
|
||||
// agree on (the host set leaves out kernel / boot / firmware names — the host's slow lane, `11` C3). A candidate is
|
||||
// APPROVED when, since it was first seen:
|
||||
// - every ring-0 box runs it (its newest report of that layer matches the candidate for every package it has),
|
||||
// - ApproveAfter (default 24 h) has passed with every ring-0 report of that layer healthy, and
|
||||
// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs of that layer.
|
||||
//
|
||||
// The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state
|
||||
// and installs exactly those versions. Rules pinned by service_test.go.
|
||||
// The operator can approve at once ("approve now"). Ring 1 gets each layer's newest release in its desired state.
|
||||
// The fleet view and the four alarms (`11` §8 step 4) live here too. Rules pinned by service_test.go and
|
||||
// alarms_test.go.
|
||||
package osupdates
|
||||
|
||||
import (
|
||||
@@ -17,21 +22,44 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// Layers.
|
||||
const (
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
)
|
||||
|
||||
// Layers lists every layer the hub approves.
|
||||
var Layers = []string{LayerGuest, LayerHost}
|
||||
|
||||
// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
|
||||
// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
|
||||
// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
|
||||
var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`)
|
||||
|
||||
// LegMinAgent is the first agent that runs the OS leg; the stale-leg alarm only watches boxes at or above it.
|
||||
const LegMinAgent = "0.140.0"
|
||||
|
||||
// Event types — operator-only except EventApplied, the household's line (`11` §5.7).
|
||||
const (
|
||||
EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed"
|
||||
EventFailed = "os_update_failed" // error, operator: the run failed or was refused
|
||||
EventHealthFailed = "os_update_health_failed" // error, operator: the guest was not healthy after the run
|
||||
EventHealthFailed = "os_update_health_failed" // error, operator: not healthy after the run
|
||||
EventReleaseApprove = "os_release_approved" // info, operator
|
||||
EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once
|
||||
EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed
|
||||
EventStale = "os_update_stale" // warning, operator: no successful OS leg for StaleAfter
|
||||
EventRebootNeeded = "os_reboot_needed" // warning, operator: reboot needed for RebootAfter
|
||||
EventRing0Stalled = "os_ring0_stalled" // error, operator: ring 0 approved nothing for Ring0StallAfter
|
||||
EventNotCovered = "os_not_covered" // warning, operator: fast-lane packages no release covers
|
||||
)
|
||||
|
||||
// Package is one name=version with its origin ("Debian" | "Debian-Security").
|
||||
@@ -41,11 +69,12 @@ type Package struct {
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
|
||||
// Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict).
|
||||
// Report is what the agent POSTs after every run of a layer.
|
||||
type Report struct {
|
||||
RunID string `json:"run_id"`
|
||||
Trigger string `json:"trigger"` // night | debug
|
||||
Mode string `json:"mode"` // apply | inventory
|
||||
Layer string `json:"layer"` // guest | host ("" from agent 0.140.0 = guest)
|
||||
Trigger string `json:"trigger"`
|
||||
Mode string `json:"mode"`
|
||||
Ring int `json:"ring"`
|
||||
ReleaseID string `json:"release_id"`
|
||||
Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed
|
||||
@@ -60,10 +89,10 @@ type Report struct {
|
||||
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
|
||||
RebootNeeded bool `json:"reboot_needed,omitempty"`
|
||||
Refused json.RawMessage `json:"refused,omitempty"`
|
||||
Log []string `json:"log,omitempty"`
|
||||
PassSeconds float64 `json:"pass_seconds,omitempty"`
|
||||
}
|
||||
|
||||
// PendingPkg is one update the guest's sources offer.
|
||||
// PendingPkg is one update the sources offer.
|
||||
type PendingPkg struct {
|
||||
Name string `json:"name"`
|
||||
From string `json:"from"`
|
||||
@@ -71,14 +100,38 @@ type PendingPkg struct {
|
||||
Origin []string `json:"origin"`
|
||||
}
|
||||
|
||||
// Block is what a box receives in its desired state (`os_update`).
|
||||
type Block struct {
|
||||
Ring int `json:"ring"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Release *ReleaseBlock `json:"release,omitempty"`
|
||||
func isFast(origins []string) bool {
|
||||
if len(origins) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, o := range origins {
|
||||
if o != "Debian" && o != "Debian-Security" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ReleaseBlock is the newest approved release, for ring 1.
|
||||
// fastPending is the pending updates a fast-lane release COULD carry for that layer.
|
||||
func fastPending(layer string, p []PendingPkg) []PendingPkg {
|
||||
var out []PendingPkg
|
||||
for _, x := range p {
|
||||
if isFast(x.Origin) && !(layer == LayerHost && hostSlowRE.MatchString(x.Name)) {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Block is what a box receives in its desired state (`os_update`).
|
||||
type Block struct {
|
||||
Ring int `json:"ring"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Release *ReleaseBlock `json:"release,omitempty"`
|
||||
HostRelease *ReleaseBlock `json:"host_release,omitempty"`
|
||||
}
|
||||
|
||||
// ReleaseBlock is a layer's newest approved release, for ring 1.
|
||||
type ReleaseBlock struct {
|
||||
ID string `json:"id"`
|
||||
Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org)
|
||||
@@ -88,12 +141,17 @@ type ReleaseBlock struct {
|
||||
// Service ties the store, the events and the clock together.
|
||||
type Service struct {
|
||||
Store *store.Store
|
||||
Emit func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests
|
||||
Emit func(customerID, eventType, severity, message, details, source string)
|
||||
ApproveAfter time.Duration
|
||||
NightsRequired int
|
||||
Logger *log.Logger
|
||||
Now func() time.Time
|
||||
Bump func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests
|
||||
// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
|
||||
StaleAfter time.Duration // 7 d
|
||||
RebootAfter time.Duration // 14 d
|
||||
Ring0StallAfter time.Duration // 7 d
|
||||
NotCoveredAfter time.Duration // 14 d
|
||||
Logger *log.Logger
|
||||
Now func() time.Time
|
||||
Bump func(hostID string)
|
||||
}
|
||||
|
||||
func (s *Service) now() time.Time {
|
||||
@@ -109,6 +167,13 @@ func (s *Service) logf(f string, a ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
func dflt(d, def time.Duration) time.Duration {
|
||||
if d <= 0 {
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (s *Service) event(customerID, typ, sev, msg string, details any) {
|
||||
dj := ""
|
||||
if details != nil {
|
||||
@@ -124,58 +189,76 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) {
|
||||
}
|
||||
}
|
||||
|
||||
func layerOf(r Report) string {
|
||||
if r.Layer == LayerHost {
|
||||
return LayerHost
|
||||
}
|
||||
return LayerGuest
|
||||
}
|
||||
|
||||
// Ingest stores a run and raises its events. The household gets one line per run that installed something.
|
||||
func (s *Service) Ingest(hostID string, r Report) error {
|
||||
h, err := s.Store.GetHost(hostID)
|
||||
if err != nil || h == nil {
|
||||
return fmt.Errorf("osupdates: unknown host %q", hostID)
|
||||
}
|
||||
layer := layerOf(r)
|
||||
r.Layer = layer
|
||||
raw, _ := json.Marshal(r)
|
||||
if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome,
|
||||
Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
|
||||
if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, Layer: layer, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode,
|
||||
Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
|
||||
return err
|
||||
}
|
||||
s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d",
|
||||
hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded))
|
||||
details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
|
||||
s.logf("[INFO] osupdates: %s reported %s run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d reboot-needed=%v wrapper=%.1fs",
|
||||
hostID, layer, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.RebootNeeded, r.PassSeconds)
|
||||
details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
|
||||
"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
|
||||
where := "the box"
|
||||
if layer == LayerHost {
|
||||
where = "the box's base system"
|
||||
}
|
||||
switch r.Outcome {
|
||||
case "applied", "health_failed":
|
||||
s.event(h.CustomerID, EventApplied, "info",
|
||||
fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details)
|
||||
fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
|
||||
if !r.Healthy || r.Outcome == "health_failed" {
|
||||
undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
|
||||
if layer == LayerHost {
|
||||
undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
|
||||
}
|
||||
s.event(h.CustomerID, EventHealthFailed, "error",
|
||||
fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.",
|
||||
hostID, len(r.Upgraded), r.HealthReason), details)
|
||||
fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
|
||||
layer, hostID, len(r.Upgraded), r.HealthReason, undo), details)
|
||||
}
|
||||
case "refused", "failed":
|
||||
s.event(h.CustomerID, EventFailed, "error",
|
||||
fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
|
||||
fmt.Sprintf("OS update (%s) on %s %s: %s", layer, hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidate derives the version set every ring-0 box agrees on, from each box's newest report.
|
||||
func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) {
|
||||
latest := map[string]*store.OSReport{}
|
||||
// candidate derives, for one layer, the version set every ring-0 box agrees on, from each box's newest report.
|
||||
func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, error) {
|
||||
byPkg := map[string]map[string]Package{} // name -> host -> pkg
|
||||
for _, h := range ring0 {
|
||||
rep, err := s.Store.LatestOSReport(h)
|
||||
rep, err := s.Store.LatestOSReport(h, layer)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
if rep == nil {
|
||||
return nil, nil, nil // a ring-0 box that never reported: no candidate
|
||||
return nil, nil // a ring-0 box that never reported this layer: no candidate
|
||||
}
|
||||
latest[h] = rep
|
||||
var r Report
|
||||
if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range r.Installed {
|
||||
if p.Origin != "Debian" && p.Origin != "Debian-Security" {
|
||||
continue
|
||||
}
|
||||
if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
|
||||
continue
|
||||
}
|
||||
if byPkg[p.Name] == nil {
|
||||
byPkg[p.Name] = map[string]Package{}
|
||||
}
|
||||
@@ -198,16 +281,17 @@ func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*sto
|
||||
cand[name] = pick
|
||||
}
|
||||
}
|
||||
return cand, latest, nil
|
||||
return cand, nil
|
||||
}
|
||||
|
||||
func fingerprint(c map[string]Package) (string, []Package) {
|
||||
func fingerprint(layer string, c map[string]Package) (string, []Package) {
|
||||
var list []Package
|
||||
for _, p := range c {
|
||||
list = append(list, p)
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name })
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "layer=%s\n", layer)
|
||||
for _, p := range list {
|
||||
fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version)
|
||||
}
|
||||
@@ -230,33 +314,45 @@ func (s *Service) ring0Hosts() ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Status explains the approval state (for the log and the fleet page).
|
||||
// Status explains the approval state of one layer (for the log and the fleet page).
|
||||
type Status struct {
|
||||
Layer string
|
||||
Fingerprint string
|
||||
FirstSeen time.Time
|
||||
Packages int
|
||||
Waiting string // why not approved yet ("" = approved or nothing to do)
|
||||
}
|
||||
|
||||
// Evaluate checks the approval rule and approves when it holds. Called every minute.
|
||||
func (s *Service) Evaluate() (Status, error) {
|
||||
// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
|
||||
func (s *Service) Evaluate() ([]Status, error) {
|
||||
var out []Status
|
||||
for _, layer := range Layers {
|
||||
st, err := s.evaluateLayer(layer)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out = append(out, st)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) evaluateLayer(layer string) (Status, error) {
|
||||
ring0, err := s.ring0Hosts()
|
||||
if err != nil || len(ring0) == 0 {
|
||||
return Status{Waiting: "no ring-0 box"}, err
|
||||
return Status{Layer: layer, Waiting: "no ring-0 box"}, err
|
||||
}
|
||||
cand, _, err := s.candidate(ring0)
|
||||
cand, err := s.candidate(layer, ring0)
|
||||
if err != nil || cand == nil {
|
||||
return Status{Waiting: "a ring-0 box has not reported"}, err
|
||||
return Status{Layer: layer, Waiting: "a ring-0 box has not reported this layer"}, err
|
||||
}
|
||||
fp, list := fingerprint(cand)
|
||||
fp, list := fingerprint(layer, cand)
|
||||
pj, _ := json.Marshal(list)
|
||||
first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
|
||||
if err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)}
|
||||
if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
|
||||
st.Waiting = ""
|
||||
st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
|
||||
return st, nil // already approved
|
||||
}
|
||||
if age := s.now().Sub(first); age < s.ApproveAfter {
|
||||
@@ -264,7 +360,7 @@ func (s *Service) Evaluate() (Status, error) {
|
||||
return st, nil
|
||||
}
|
||||
for _, h := range ring0 {
|
||||
reps, err := s.Store.OSReportsSince(h, first)
|
||||
reps, err := s.Store.OSReportsSince(h, layer, first)
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
@@ -283,49 +379,54 @@ func (s *Service) Evaluate() (Status, error) {
|
||||
return st, nil
|
||||
}
|
||||
}
|
||||
if err := s.approve(fp, list, "auto"); err != nil {
|
||||
return st, err
|
||||
}
|
||||
return st, nil
|
||||
return st, s.approve(layer, fp, list, "auto")
|
||||
}
|
||||
|
||||
// ApproveNow approves the current candidate at once (operator, urgent fix).
|
||||
// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
|
||||
func (s *Service) ApproveNow() (string, error) {
|
||||
ring0, err := s.ring0Hosts()
|
||||
if err != nil || len(ring0) == 0 {
|
||||
return "", fmt.Errorf("osupdates: no ring-0 box")
|
||||
}
|
||||
cand, _, err := s.candidate(ring0)
|
||||
if err != nil || cand == nil {
|
||||
var ids []string
|
||||
for _, layer := range Layers {
|
||||
cand, err := s.candidate(layer, ring0)
|
||||
if err != nil || cand == nil {
|
||||
continue
|
||||
}
|
||||
fp, list := fingerprint(layer, cand)
|
||||
pj, _ := json.Marshal(list)
|
||||
if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
|
||||
ids = append(ids, rel.ID)
|
||||
continue
|
||||
}
|
||||
if err := s.approve(layer, fp, list, "operator"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, _ := s.Store.LatestOSRelease(layer)
|
||||
s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s (%s) at once (%d packages), without the wait.", rel.ID, layer, len(list)),
|
||||
map[string]any{"release_id": rel.ID, "layer": layer, "packages": len(list)})
|
||||
ids = append(ids, rel.ID)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet")
|
||||
}
|
||||
fp, list := fingerprint(cand)
|
||||
pj, _ := json.Marshal(list)
|
||||
if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
|
||||
return rel.ID, nil
|
||||
}
|
||||
if err := s.approve(fp, list, "operator"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, _ := s.Store.LatestOSRelease()
|
||||
s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)),
|
||||
map[string]any{"release_id": rel.ID, "packages": len(list)})
|
||||
return rel.ID, nil
|
||||
return strings.Join(ids, ","), nil
|
||||
}
|
||||
|
||||
func (s *Service) approve(fp string, list []Package, by string) error {
|
||||
func (s *Service) approve(layer, fp string, list []Package, by string) error {
|
||||
at := s.now().UTC().Truncate(time.Second)
|
||||
id := "os-" + at.Format("20060102-150405")
|
||||
id := "os-" + layer + "-" + at.Format("20060102-150405")
|
||||
pj, _ := json.Marshal(list)
|
||||
if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
|
||||
if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
|
||||
return err
|
||||
}
|
||||
s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp)
|
||||
s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)),
|
||||
map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp})
|
||||
s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
|
||||
s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
|
||||
map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
|
||||
if s.Bump != nil {
|
||||
hosts, _ := s.Store.ListHosts()
|
||||
for _, h := range hosts {
|
||||
@@ -337,17 +438,24 @@ func (s *Service) approve(fp string, list []Package, by string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) releaseBlock(layer string) *ReleaseBlock {
|
||||
rel, err := s.Store.LatestOSRelease(layer)
|
||||
if err != nil || rel == nil {
|
||||
return nil
|
||||
}
|
||||
var list []Package
|
||||
if json.Unmarshal([]byte(rel.PackagesJSON), &list) != nil {
|
||||
return nil
|
||||
}
|
||||
return &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
|
||||
}
|
||||
|
||||
// DesiredBlock is the `os_update` block a box receives.
|
||||
func (s *Service) DesiredBlock(hostID string) Block {
|
||||
st := s.Store.GetOSHostSettings(hostID)
|
||||
b := Block{Ring: st.Ring, Enabled: st.Enabled}
|
||||
if st.Ring == 1 {
|
||||
if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil {
|
||||
var list []Package
|
||||
if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil {
|
||||
b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
|
||||
}
|
||||
}
|
||||
b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost)
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -387,17 +495,98 @@ func (s *Service) SetEnabled(hostID string, on bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FleetLine is one box on the fleet page.
|
||||
// LayerLine is one layer of one box on the fleet view.
|
||||
type LayerLine struct {
|
||||
ReleaseID string
|
||||
LastOutcome string
|
||||
LastAt time.Time
|
||||
LastSuccessfulLeg time.Time
|
||||
Pending int
|
||||
NotCovered int
|
||||
NotCoveredFast int // fast-lane packages no approved release covers (the alarm's input)
|
||||
RestartNeeded int
|
||||
RebootNeededSince time.Time // zero = not needed
|
||||
WrapperPassSeconds float64
|
||||
}
|
||||
|
||||
// FleetLine is one box on the fleet view.
|
||||
type FleetLine struct {
|
||||
HostID string
|
||||
Ring int
|
||||
Enabled bool
|
||||
ReleaseID string
|
||||
LastOutcome string
|
||||
LastAt time.Time
|
||||
Pending int
|
||||
NotCovered int
|
||||
RestartNeeded int
|
||||
HostID string
|
||||
Ring int
|
||||
Enabled bool
|
||||
Tunnel string
|
||||
Guest LayerLine
|
||||
Host LayerLine
|
||||
}
|
||||
|
||||
func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
|
||||
var l LayerLine
|
||||
rep, _ := s.Store.LatestOSReport(hostID, layer)
|
||||
if rep == nil {
|
||||
return l
|
||||
}
|
||||
var r Report
|
||||
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
|
||||
l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
|
||||
l.Pending, l.NotCovered, l.RestartNeeded, l.WrapperPassSeconds = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.PassSeconds
|
||||
if ok, _ := s.Store.LatestHealthyOSReport(hostID, layer, "applied", "nothing"); ok != nil {
|
||||
l.LastSuccessfulLeg = ok.ReceivedAt
|
||||
}
|
||||
l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
|
||||
l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) {
|
||||
if len(x.Upgraded) == 0 && x.Outcome != "applied" {
|
||||
return false, false // a report without an install says nothing about reboot-needed: skip it
|
||||
}
|
||||
return x.RebootNeeded, true
|
||||
})
|
||||
return l
|
||||
}
|
||||
|
||||
// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
|
||||
// installs every fast-lane update itself, so it never has any.
|
||||
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
|
||||
if ring == 0 {
|
||||
return nil
|
||||
}
|
||||
named := map[string]bool{}
|
||||
if rb := s.releaseBlock(layer); rb != nil {
|
||||
for _, p := range rb.Packages {
|
||||
named[p.Name] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, p := range fastPending(layer, r.Pending) {
|
||||
if !named[p.Name] {
|
||||
out = append(out, p.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sinceContinuously walks the box's reports of a layer newest-first and returns the time of the OLDEST report in the
|
||||
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
|
||||
// false = the report does not count either way (skipped).
|
||||
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
|
||||
reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
var since time.Time
|
||||
for _, rep := range reps {
|
||||
var r Report
|
||||
if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
|
||||
continue
|
||||
}
|
||||
hold, counts := pred(r)
|
||||
if !counts {
|
||||
continue
|
||||
}
|
||||
if !hold {
|
||||
break
|
||||
}
|
||||
since = rep.ReceivedAt
|
||||
}
|
||||
return since
|
||||
}
|
||||
|
||||
// Fleet lists every box.
|
||||
@@ -409,13 +598,12 @@ func (s *Service) Fleet() ([]FleetLine, error) {
|
||||
var out []FleetLine
|
||||
for _, h := range hosts {
|
||||
st := s.Store.GetOSHostSettings(h.HostID)
|
||||
l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled}
|
||||
if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil {
|
||||
var r Report
|
||||
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
|
||||
l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
|
||||
l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)
|
||||
l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled, Tunnel: "unknown"}
|
||||
if t, _ := s.Store.RecentCloudflaredStatuses(h.HostID, 1); len(t) == 1 && t[0] != "" {
|
||||
l.Tunnel = t[0]
|
||||
}
|
||||
l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
|
||||
l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
|
||||
out = append(out, l)
|
||||
}
|
||||
return out, nil
|
||||
@@ -427,10 +615,137 @@ func (s *Service) FleetJSON() (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rel, _ := s.Store.LatestOSRelease()
|
||||
out := map[string]any{"boxes": lines}
|
||||
if rel != nil {
|
||||
out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
|
||||
for _, layer := range Layers {
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
|
||||
out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ---------------- the alarms (`11` §8.3) ----------------
|
||||
|
||||
// raise sends an alarm when its condition holds and it was not raised in the last week; clears it when it does not
|
||||
// hold, so the next occurrence is announced again. Returns true when an event was sent.
|
||||
func (s *Service) raise(key string, holds bool, customerID, typ, sev, msg string, details any) bool {
|
||||
last := s.Store.OSAlarmRaised(key)
|
||||
if !holds {
|
||||
if !last.IsZero() {
|
||||
_ = s.Store.SetOSAlarmRaised(key, time.Time{})
|
||||
}
|
||||
return false
|
||||
}
|
||||
if !last.IsZero() && s.now().Sub(last) < 7*24*time.Hour {
|
||||
return false
|
||||
}
|
||||
s.event(customerID, typ, sev, msg, details)
|
||||
_ = s.Store.SetOSAlarmRaised(key, s.now())
|
||||
return true
|
||||
}
|
||||
|
||||
// Alarms evaluates the four OS-update alarms. Called hourly. Returns the event types it sent (for the log/tests).
|
||||
func (s *Service) Alarms() ([]string, error) {
|
||||
now := s.now()
|
||||
stale, reboot, stall, nc := dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour),
|
||||
dflt(s.Ring0StallAfter, 7*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
|
||||
hosts, err := s.Store.ListHosts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var sent []string
|
||||
for _, h := range hosts {
|
||||
st := s.Store.GetOSHostSettings(h.HostID)
|
||||
// 1. No successful OS leg for `stale` while the switch is ON — the leg runs only after a good night backup,
|
||||
// so a box with a broken backup silently stops updating. Only boxes whose agent CAN run the leg.
|
||||
canRun := h.AgentVersion != "" && semver.Valid(h.AgentVersion) && semver.Compare(h.AgentVersion, LegMinAgent) >= 0
|
||||
var ref time.Time
|
||||
if ok, _ := s.Store.LatestHealthyOSReport(h.HostID, LayerGuest, "applied", "nothing"); ok != nil {
|
||||
ref = ok.ReceivedAt
|
||||
} else if seen, _ := s.Store.HostAgentVersionsSeen(h.HostID); len(seen) > 0 {
|
||||
for v, at := range seen {
|
||||
if semver.Valid(v) && semver.Compare(v, LegMinAgent) >= 0 && (ref.IsZero() || at.Before(ref)) {
|
||||
ref = at
|
||||
}
|
||||
}
|
||||
}
|
||||
holds := st.Enabled && canRun && !ref.IsZero() && now.Sub(ref) >= stale
|
||||
if s.raise("stale:"+h.HostID, holds, h.CustomerID, EventStale, "warning",
|
||||
fmt.Sprintf("OS updates: %s has not completed an OS update leg for %s (since %s). Likely reason: %s.",
|
||||
h.HostID, now.Sub(ref).Round(time.Hour), ref.UTC().Format("2006-01-02 15:04"), s.staleReason(h)),
|
||||
map[string]any{"host_id": h.HostID, "since": ref}) {
|
||||
sent = append(sent, EventStale)
|
||||
}
|
||||
// 2. Reboot needed for `reboot` (host layer): a fix is installed but not fully in force.
|
||||
since := s.layerLine(h.HostID, LayerHost, st.Ring).RebootNeededSince
|
||||
if s.raise("reboot:"+h.HostID, !since.IsZero() && now.Sub(since) >= reboot, h.CustomerID, EventRebootNeeded, "warning",
|
||||
fmt.Sprintf("OS updates (for information): %s has needed a reboot since %s — host fixes are installed but not fully in force (PID 1 or lxc-start still run the old libraries). Nothing reboots automatically until the kernel lane exists.",
|
||||
h.HostID, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "since": since}) {
|
||||
sent = append(sent, EventRebootNeeded)
|
||||
}
|
||||
// 4. Not-covered fast-lane packages for `nc` (ring 1: hardware ring 0 does not have, `11` edge case 9).
|
||||
for _, layer := range Layers {
|
||||
since := s.sinceContinuously(h.HostID, layer, func(r Report) (bool, bool) {
|
||||
return len(s.notCoveredFast(layer, st.Ring, r)) > 0, true
|
||||
})
|
||||
var names []string
|
||||
if rep, _ := s.Store.LatestOSReport(h.HostID, layer); rep != nil {
|
||||
var r Report
|
||||
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
|
||||
names = s.notCoveredFast(layer, st.Ring, r)
|
||||
}
|
||||
if s.raise("notcovered:"+layer+":"+h.HostID, !since.IsZero() && now.Sub(since) >= nc, h.CustomerID, EventNotCovered, "warning",
|
||||
fmt.Sprintf("OS updates (%s): %s has had %d fast-lane package(s) no approved release covers since %s: %s. Ring 0 does not run them (other hardware?) — add matching hardware to ring 0, or approve by hand.",
|
||||
layer, h.HostID, len(names), since.UTC().Format("2006-01-02"), strings.Join(names, ", ")),
|
||||
map[string]any{"host_id": h.HostID, "layer": layer, "packages": names}) {
|
||||
sent = append(sent, EventNotCovered)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped
|
||||
// getting fixes.
|
||||
ring0, _ := s.ring0Hosts()
|
||||
for _, layer := range Layers {
|
||||
var last time.Time
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
|
||||
last = rel.ApprovedAt
|
||||
} else if first, _ := s.Store.FirstOSReport(layer, ring0); first != nil {
|
||||
last = first.ReceivedAt
|
||||
}
|
||||
pendingAny := 0
|
||||
for _, hID := range ring0 {
|
||||
if rep, _ := s.Store.LatestOSReport(hID, layer); rep != nil {
|
||||
var r Report
|
||||
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
|
||||
pendingAny += len(fastPending(layer, r.Pending))
|
||||
}
|
||||
}
|
||||
holds := len(ring0) > 0 && !last.IsZero() && now.Sub(last) >= stall && pendingAny > 0
|
||||
if s.raise("ring0stall:"+layer, holds, "", EventRing0Stalled, "error",
|
||||
fmt.Sprintf("OS updates (%s): ring 0 has approved nothing since %s while it has %d pending fast-lane update(s). The demo boxes are broken or unhealthy, and the whole fleet has stopped getting fixes.",
|
||||
layer, last.UTC().Format("2006-01-02 15:04"), pendingAny), map[string]any{"layer": layer, "pending": pendingAny}) {
|
||||
sent = append(sent, EventRing0Stalled)
|
||||
}
|
||||
}
|
||||
return sent, nil
|
||||
}
|
||||
|
||||
// staleReason names the likely reason a box stopped completing the OS leg (for the operator's mail).
|
||||
func (s *Service) staleReason(h store.Host) string {
|
||||
if h.LastReportAt == nil || s.now().Sub(*h.LastReportAt) > 24*time.Hour {
|
||||
return "the box is off or not reporting (last host report " + fmtTime(h.LastReportAt) + ")"
|
||||
}
|
||||
if rep, _ := s.Store.LatestOSReport(h.HostID, LayerGuest); rep != nil && (rep.Outcome == "failed" || rep.Outcome == "refused" || rep.Outcome == "health_failed" || !rep.Healthy) {
|
||||
var r Report
|
||||
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
|
||||
return fmt.Sprintf("the last OS leg ended %s (%s) on %s", rep.Outcome, strings.TrimSpace(r.HealthReason+" "+string(r.Refused)), rep.ReceivedAt.UTC().Format("2006-01-02"))
|
||||
}
|
||||
return "no successful whole-guest backup reached the OS leg (it runs only after one) — check the box's night backups"
|
||||
}
|
||||
|
||||
func fmtTime(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "never"
|
||||
}
|
||||
return t.UTC().Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
@@ -42,12 +42,17 @@ func newFix(t *testing.T) *fix {
|
||||
func pk(name, ver string) Package { return Package{Name: name, Version: ver, Origin: "Debian"} }
|
||||
|
||||
func (f *fix) report(t *testing.T, host, trigger string, healthy bool, pkgs ...Package) {
|
||||
t.Helper()
|
||||
f.reportL(t, host, LayerGuest, trigger, healthy, pkgs...)
|
||||
}
|
||||
|
||||
func (f *fix) reportL(t *testing.T, host, layer, trigger string, healthy bool, pkgs ...Package) {
|
||||
t.Helper()
|
||||
outcome := "applied"
|
||||
if !healthy {
|
||||
outcome = "health_failed"
|
||||
}
|
||||
if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Trigger: trigger, Mode: "apply", Outcome: outcome,
|
||||
if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Layer: layer, Trigger: trigger, Mode: "apply", Outcome: outcome,
|
||||
Healthy: healthy, Installed: pkgs, Upgraded: pkgs[:1]}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -60,12 +65,14 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) {
|
||||
set := []Package{pk("libc6", "2.41-12+deb13u4"), pk("openssl", "3.5.7-1~deb13u3")}
|
||||
f.report(t, "hp", "debug", true, set...)
|
||||
f.report(t, "n100", "debug", true, set...)
|
||||
st, _ := f.s.Evaluate()
|
||||
sts, _ := f.s.Evaluate()
|
||||
st := sts[0]
|
||||
if !strings.HasPrefix(st.Waiting, "healthy for") {
|
||||
t.Fatalf("fresh set approved or wrong reason: %+v", st)
|
||||
}
|
||||
f.now = f.now.Add(25 * time.Hour)
|
||||
st, _ = f.s.Evaluate()
|
||||
sts, _ = f.s.Evaluate()
|
||||
st = sts[0]
|
||||
if !strings.Contains(st.Waiting, "night run") {
|
||||
t.Fatalf("approved without a night run: %+v", st)
|
||||
}
|
||||
@@ -74,7 +81,7 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) {
|
||||
if _, err := f.s.Evaluate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rel, _ := f.s.Store.LatestOSRelease()
|
||||
rel, _ := f.s.Store.LatestOSRelease(LayerGuest)
|
||||
if rel == nil || rel.ApprovedBy != "auto" {
|
||||
t.Fatalf("not approved: %+v", rel)
|
||||
}
|
||||
@@ -85,6 +92,9 @@ func TestApproval_WaitHealthyAndOneNight(t *testing.T) {
|
||||
if b.Ring != 1 || !b.Enabled || b.Release == nil || len(b.Release.Packages) != 2 || b.Release.Snapshot != rel.ApprovedAt.UTC().Format("20060102T150405Z") {
|
||||
t.Fatalf("ring-1 block = %+v", b)
|
||||
}
|
||||
if b.HostRelease != nil {
|
||||
t.Fatalf("a guest approval must not create a host release (separate sets): %+v", b.HostRelease)
|
||||
}
|
||||
if hb := f.s.DesiredBlock("hp"); hb.Ring != 0 || hb.Release != nil {
|
||||
t.Fatalf("a ring-0 box must not get a release (it installs everything pending): %+v", hb)
|
||||
}
|
||||
@@ -99,8 +109,9 @@ func TestApproval_UnhealthyRunBlocks(t *testing.T) {
|
||||
f.now = f.now.Add(25 * time.Hour)
|
||||
f.report(t, "hp", "night", false, set...)
|
||||
f.report(t, "n100", "night", true, set...)
|
||||
st, _ := f.s.Evaluate()
|
||||
if rel, _ := f.s.Store.LatestOSRelease(); rel != nil {
|
||||
sts, _ := f.s.Evaluate()
|
||||
st := sts[0]
|
||||
if rel, _ := f.s.Store.LatestOSRelease(LayerGuest); rel != nil {
|
||||
t.Fatalf("approved although a ring-0 run was not healthy: %+v", st)
|
||||
}
|
||||
if !strings.Contains(st.Waiting, "healthy=false") {
|
||||
@@ -120,7 +131,7 @@ func TestCandidate_DisagreementLeftOut(t *testing.T) {
|
||||
f := newFix(t)
|
||||
f.report(t, "hp", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u5"))
|
||||
f.report(t, "n100", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u4"))
|
||||
cand, _, err := f.s.candidate([]string{"hp", "n100"})
|
||||
cand, err := f.s.candidate(LayerGuest, []string{"hp", "n100"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -135,7 +146,7 @@ func TestCandidate_OnlyDebianOrigins(t *testing.T) {
|
||||
d := Package{Name: "docker-ce", Version: "5:29.8.2", Origin: "Docker"}
|
||||
f.report(t, "hp", "night", true, pk("libc6", "x1"), d)
|
||||
f.report(t, "n100", "night", true, pk("libc6", "x1"), d)
|
||||
cand, _, _ := f.s.candidate([]string{"hp", "n100"})
|
||||
cand, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"})
|
||||
if _, ok := cand["docker-ce"]; ok {
|
||||
t.Fatal("a Docker package entered the candidate")
|
||||
}
|
||||
@@ -149,7 +160,7 @@ func TestApproveNow_IsAnOperatorEvent(t *testing.T) {
|
||||
if err != nil || id == "" {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
rel, _ := f.s.Store.LatestOSRelease()
|
||||
rel, _ := f.s.Store.LatestOSRelease(LayerGuest)
|
||||
if rel.ApprovedBy != "operator" {
|
||||
t.Fatalf("approved_by = %q", rel.ApprovedBy)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -46,7 +47,13 @@ func (s *Store) migrateOSUpdates() error {
|
||||
packages_json TEXT NOT NULL
|
||||
);
|
||||
`)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Host fast lane (hub v0.131.0): every report and release belongs to a LAYER; old rows are the guest's.
|
||||
s.db.Exec(`ALTER TABLE os_reports ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`)
|
||||
s.db.Exec(`ALTER TABLE os_releases ADD COLUMN layer TEXT NOT NULL DEFAULT 'guest'`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// OSHostSettings is one box's ring and switch.
|
||||
@@ -88,6 +95,7 @@ func (s *Store) SetOSEnabled(hostID string, on bool) error {
|
||||
type OSReport struct {
|
||||
ID int64
|
||||
HostID string
|
||||
Layer string // guest | host
|
||||
ReceivedAt time.Time
|
||||
Trigger string
|
||||
Mode string
|
||||
@@ -109,8 +117,12 @@ func (s *Store) SaveOSReport(r OSReport) (int64, error) {
|
||||
}
|
||||
// received_at comes from the CALLER's clock: the approval rule compares it with first_seen, which the
|
||||
// service stamps with its own clock — two clocks would make "since first seen" miss reports.
|
||||
res, err := s.db.Exec(`INSERT INTO os_reports (host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
r.HostID, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON)
|
||||
layer := r.Layer
|
||||
if layer == "" {
|
||||
layer = "guest"
|
||||
}
|
||||
res, err := s.db.Exec(`INSERT INTO os_reports (host_id, layer, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
r.HostID, layer, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -124,7 +136,7 @@ func scanOSReports(rows *sql.Rows) ([]OSReport, error) {
|
||||
var r OSReport
|
||||
var at string
|
||||
var h int
|
||||
if err := rows.Scan(&r.ID, &r.HostID, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil {
|
||||
if err := rows.Scan(&r.ID, &r.HostID, &r.Layer, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.ReceivedAt, r.Healthy = parseSQLiteTime(at), h == 1
|
||||
@@ -133,11 +145,11 @@ func scanOSReports(rows *sql.Rows) ([]OSReport, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
const osReportCols = `id, host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json`
|
||||
const osReportCols = `id, host_id, layer, received_at, trigger, mode, outcome, healthy, release_id, report_json`
|
||||
|
||||
// LatestOSReport returns the box's newest run, or nil.
|
||||
func (s *Store) LatestOSReport(hostID string) (*OSReport, error) {
|
||||
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? ORDER BY id DESC LIMIT 1`, hostID)
|
||||
// LatestOSReport returns the box's newest run for a layer, or nil.
|
||||
func (s *Store) LatestOSReport(hostID, layer string) (*OSReport, error) {
|
||||
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? ORDER BY id DESC LIMIT 1`, hostID, layer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -148,10 +160,10 @@ func (s *Store) LatestOSReport(hostID string) (*OSReport, error) {
|
||||
return &rs[0], nil
|
||||
}
|
||||
|
||||
// OSReportsSince returns the box's runs received at or after t, oldest first.
|
||||
func (s *Store) OSReportsSince(hostID string, t time.Time) ([]OSReport, error) {
|
||||
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND received_at >= ? ORDER BY id`,
|
||||
hostID, t.UTC().Format("2006-01-02 15:04:05"))
|
||||
// OSReportsSince returns the box's runs of a layer received at or after t, oldest first.
|
||||
func (s *Store) OSReportsSince(hostID, layer string, t time.Time) ([]OSReport, error) {
|
||||
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? AND received_at >= ? ORDER BY id`,
|
||||
hostID, layer, t.UTC().Format("2006-01-02 15:04:05"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -174,6 +186,7 @@ func (s *Store) OSCandidateFirstSeen(fingerprint, packagesJSON string, now time.
|
||||
// OSRelease is one approved version set.
|
||||
type OSRelease struct {
|
||||
ID string
|
||||
Layer string
|
||||
Fingerprint string
|
||||
ApprovedAt time.Time
|
||||
ApprovedBy string
|
||||
@@ -182,17 +195,21 @@ type OSRelease struct {
|
||||
|
||||
// SaveOSRelease stores an approved release.
|
||||
func (s *Store) SaveOSRelease(r OSRelease) error {
|
||||
_, err := s.db.Exec(`INSERT INTO os_releases (id, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?)`,
|
||||
r.ID, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON)
|
||||
layer := r.Layer
|
||||
if layer == "" {
|
||||
layer = "guest"
|
||||
}
|
||||
_, err := s.db.Exec(`INSERT INTO os_releases (id, layer, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
r.ID, layer, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON)
|
||||
return err
|
||||
}
|
||||
|
||||
// LatestOSRelease returns the newest approved release, or nil.
|
||||
func (s *Store) LatestOSRelease() (*OSRelease, error) {
|
||||
// LatestOSRelease returns the newest approved release of a layer, or nil.
|
||||
func (s *Store) LatestOSRelease(layer string) (*OSRelease, error) {
|
||||
var r OSRelease
|
||||
var at string
|
||||
err := s.db.QueryRow(`SELECT id, fingerprint, approved_at, approved_by, packages_json FROM os_releases ORDER BY approved_at DESC, id DESC LIMIT 1`).
|
||||
Scan(&r.ID, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON)
|
||||
err := s.db.QueryRow(`SELECT id, layer, fingerprint, approved_at, approved_by, packages_json FROM os_releases WHERE layer = ? ORDER BY approved_at DESC, id DESC LIMIT 1`, layer).
|
||||
Scan(&r.ID, &r.Layer, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -209,3 +226,90 @@ func (s *Store) BackdateOSCandidateForTest(fingerprint string, by time.Duration)
|
||||
time.Now().Add(-by).UTC().Format("2006-01-02 15:04:05"), fingerprint)
|
||||
return err
|
||||
}
|
||||
|
||||
// OSAlarmRaised returns when an OS alarm (by key) was last raised; zero = not raised / cleared (hub v0.131.0).
|
||||
func (s *Store) OSAlarmRaised(key string) time.Time {
|
||||
v := s.getSetting("os_alarm:" + key)
|
||||
if v == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
t, _ := time.Parse(time.RFC3339, v)
|
||||
return t
|
||||
}
|
||||
|
||||
// SetOSAlarmRaised records (t non-zero) or clears (zero t) an OS alarm.
|
||||
func (s *Store) SetOSAlarmRaised(key string, t time.Time) error {
|
||||
v := ""
|
||||
if !t.IsZero() {
|
||||
v = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return s.setSetting("os_alarm:"+key, v)
|
||||
}
|
||||
|
||||
// HostAgentVersionsSeen returns, per agent version a host ever reported, the first time it was seen (the stale-leg
|
||||
// alarm's start point for a box whose agent CAN run the OS leg but never reported one).
|
||||
func (s *Store) HostAgentVersionsSeen(hostID string) (map[string]time.Time, error) {
|
||||
rows, err := s.db.Query(`SELECT COALESCE(agent_version, ''), MIN(received_at) FROM host_reports WHERE host_id = ? GROUP BY agent_version`, hostID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]time.Time{}
|
||||
for rows.Next() {
|
||||
var v, at string
|
||||
if err := rows.Scan(&v, &at); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[v] = parseSQLiteTime(at)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LatestOSReportWhere returns the box's newest report of a layer whose outcome is one of outcomes and healthy, or nil.
|
||||
func (s *Store) LatestHealthyOSReport(hostID, layer string, outcomes ...string) (*OSReport, error) {
|
||||
q := `SELECT ` + osReportCols + ` FROM os_reports WHERE host_id = ? AND layer = ? AND healthy = 1 AND outcome IN (?` +
|
||||
strings.Repeat(",?", len(outcomes)-1) + `) ORDER BY id DESC LIMIT 1`
|
||||
args := []any{hostID, layer}
|
||||
for _, o := range outcomes {
|
||||
args = append(args, o)
|
||||
}
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rs, err := scanOSReports(rows)
|
||||
if err != nil || len(rs) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
return &rs[0], nil
|
||||
}
|
||||
|
||||
// OSReportsDesc returns the box's newest n reports of a layer, newest first.
|
||||
func (s *Store) OSReportsDesc(hostID, layer string, n int) ([]OSReport, error) {
|
||||
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND layer = ? ORDER BY id DESC LIMIT ?`, hostID, layer, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanOSReports(rows)
|
||||
}
|
||||
|
||||
// FirstOSReport returns the oldest report of a layer from any of the given hosts, or nil.
|
||||
func (s *Store) FirstOSReport(layer string, hosts []string) (*OSReport, error) {
|
||||
if len(hosts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
q := `SELECT ` + osReportCols + ` FROM os_reports WHERE layer = ? AND host_id IN (?` + strings.Repeat(",?", len(hosts)-1) + `) ORDER BY id LIMIT 1`
|
||||
args := []any{layer}
|
||||
for _, h := range hosts {
|
||||
args = append(args, h)
|
||||
}
|
||||
rows, err := s.db.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rs, err := scanOSReports(rows)
|
||||
if err != nil || len(rs) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
return &rs[0], nil
|
||||
}
|
||||
|
||||
@@ -3442,6 +3442,24 @@ func (s *Store) GetHostDRBundle(hostID string) (*HostDRBundle, error) {
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// RecentCloudflaredStatuses returns the host's last n reported tunnel states, newest first (R-841).
|
||||
func (s *Store) RecentCloudflaredStatuses(hostID string, n int) ([]string, error) {
|
||||
rows, err := s.db.Query(`SELECT COALESCE(cloudflared_status, '') FROM host_reports WHERE host_id = ? ORDER BY id DESC LIMIT ?`, hostID, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var st string
|
||||
if err := rows.Scan(&st); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, st)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SaveHostReport inserts a host_reports row and bumps the host's reality columns
|
||||
// (agent_version/last_report_at/updated_at) — never the inert intent columns.
|
||||
func (s *Store) SaveHostReport(hostID, customerID string, reportJSON []byte, d HostReportDenorm) error {
|
||||
|
||||
@@ -79,6 +79,7 @@ type hostVitals struct {
|
||||
MemoryPercent float64
|
||||
DiskPercent float64
|
||||
CloudflaredStatus string
|
||||
CloudflaredDetail string // agent >= 0.141.0: why not_running (container state, exit code, readiness)
|
||||
}
|
||||
|
||||
// parseHostVitals extracts the vitals block from a host-report body. A missing/malformed
|
||||
@@ -96,6 +97,7 @@ func parseHostVitals(reportJSON string) hostVitals {
|
||||
} `json:"host"`
|
||||
Cloudflared struct {
|
||||
Status string `json:"status"`
|
||||
Detail string `json:"detail"`
|
||||
} `json:"cloudflared"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(reportJSON), &body); err != nil {
|
||||
@@ -105,6 +107,7 @@ func parseHostVitals(reportJSON string) hostVitals {
|
||||
v.MemoryPercent = body.Host.MemoryPercent
|
||||
v.DiskPercent = body.Host.DiskPercent
|
||||
v.CloudflaredStatus = body.Cloudflared.Status
|
||||
v.CloudflaredDetail = body.Cloudflared.Detail
|
||||
return v
|
||||
}
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
</div>
|
||||
<div class="info-item">
|
||||
<span class="label">Cloudflared</span>
|
||||
<span class="value">{{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{else}}—{{end}}</span>
|
||||
<span class="value">{{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{if .Vitals.CloudflaredDetail}} ({{.Vitals.CloudflaredDetail}}){{end}}{{else}}—{{end}}</span>
|
||||
</div>
|
||||
<div class="info-item">
|
||||
<span class="label">Guests</span>
|
||||
|
||||
@@ -105,7 +105,7 @@
|
||||
<td>{{if .HasReport}}{{formatFloat .Vitals.CPUPercent}}%{{else}}—{{end}}</td>
|
||||
<td>{{if .HasReport}}{{formatFloat .Vitals.MemoryPercent}}%{{else}}—{{end}}</td>
|
||||
<td>{{if .HasReport}}{{formatFloat .Vitals.DiskPercent}}%{{else}}—{{end}}</td>
|
||||
<td>{{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{else}}—{{end}}</td>
|
||||
<td>{{if .Vitals.CloudflaredStatus}}{{.Vitals.CloudflaredStatus}}{{if .Vitals.CloudflaredDetail}} ({{.Vitals.CloudflaredDetail}}){{end}}{{else}}—{{end}}</td>
|
||||
<td>{{if .HasStorage}}{{formatFloat .WorstFillPct}}% <span class="text-muted">{{.WorstFillName}}</span>{{else}}—{{end}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
|
||||
Reference in New Issue
Block a user