hub v0.131.0: the tunnel status is true (R-841: three states, tunnel_down after two not_running reports, unknown never alarms); OS updates per layer (guest/host separate approved sets, host candidate leaves out kernel/boot/firmware, host_release in the box block), the fleet view and four hourly operator alarms (11 §8 steps 3+4); red-proofs in audits/os-host-lane-2026-10-04
gates / gates (push) Successful in 30s
gates / gates (push) Successful in 30s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+40
-6
@@ -414,23 +414,57 @@ func main() {
|
||||
logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired)
|
||||
apiHandler.SetOSUpdateService(osSvc)
|
||||
webServer.SetOSUpdateAdmin(osSvc)
|
||||
// `11` §8.3: the four alarm thresholds are configuration (decided by CC unattended — operator may reverse).
|
||||
for _, a := range []struct {
|
||||
env string
|
||||
dst *time.Duration
|
||||
def time.Duration
|
||||
}{
|
||||
{"OS_ALARM_STALE_AFTER", &osSvc.StaleAfter, 7 * 24 * time.Hour},
|
||||
{"OS_ALARM_REBOOT_AFTER", &osSvc.RebootAfter, 14 * 24 * time.Hour},
|
||||
{"OS_ALARM_RING0_STALL_AFTER", &osSvc.Ring0StallAfter, 7 * 24 * time.Hour},
|
||||
{"OS_ALARM_NOT_COVERED_AFTER", &osSvc.NotCoveredAfter, 14 * 24 * time.Hour},
|
||||
} {
|
||||
*a.dst = a.def
|
||||
if v := os.Getenv(a.env); v != "" {
|
||||
if d, derr := time.ParseDuration(v); derr == nil && d > 0 {
|
||||
*a.dst = d
|
||||
} else {
|
||||
logger.Printf("[ERROR] %s=%q invalid — keeping %s", a.env, v, a.def)
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Printf("[INFO] osupdates: alarms = no OS leg %s, reboot needed %s, ring 0 stalled %s, not covered %s",
|
||||
osSvc.StaleAfter, osSvc.RebootAfter, osSvc.Ring0StallAfter, osSvc.NotCoveredAfter)
|
||||
go func() {
|
||||
tk := time.NewTicker(60 * time.Second)
|
||||
defer tk.Stop()
|
||||
last := ""
|
||||
last := map[string]string{}
|
||||
var lastAlarms time.Time
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tk.C:
|
||||
st, err := osSvc.Evaluate()
|
||||
sts, err := osSvc.Evaluate()
|
||||
if err != nil {
|
||||
logger.Printf("[WARN] osupdates: evaluate: %v", err)
|
||||
continue
|
||||
}
|
||||
if msg := st.Fingerprint + "|" + st.Waiting; msg != last {
|
||||
last = msg
|
||||
logger.Printf("[INFO] osupdates: candidate %s (%d packages, first seen %s): %s", st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""])
|
||||
for _, st := range sts {
|
||||
if msg := st.Fingerprint + "|" + st.Waiting; msg != last[st.Layer] {
|
||||
last[st.Layer] = msg
|
||||
logger.Printf("[INFO] osupdates: %s candidate %s (%d packages, first seen %s): %s", st.Layer, st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""])
|
||||
}
|
||||
}
|
||||
if time.Since(lastAlarms) >= time.Hour {
|
||||
lastAlarms = time.Now()
|
||||
start := time.Now()
|
||||
sent, aerr := osSvc.Alarms()
|
||||
if aerr != nil {
|
||||
logger.Printf("[WARN] osupdates: alarms: %v", aerr)
|
||||
} else {
|
||||
logger.Printf("[INFO] osupdates: alarm check done in %s, %d alarm(s) sent %v", time.Since(start).Round(time.Millisecond), len(sent), sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user