Files
felhom.eu/hub/internal/monitor/signed_job_waits.go
T
admin 1886030782
gates / gates (push) Successful in 2m9s
hub (unreleased): four silent failures tell the operator (R-1383, D78)
A hub daily check skipped while the hub was down; an operator mail that was never sent; a signed job a box did not
take; a signed job a box refused, and the guest-power watchdog's give-up (both from the agent). Operator-only.
No new table. Not deployed.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-11-14 17:29:53 +01:00

77 lines
3.2 KiB
Go

package monitor
import (
"encoding/json"
"fmt"
"log"
"time"
)
// SignedJobWaitChecker (R-1383, D78, `09` §3 decision 299) tells the operator when a signed job is still in a box's
// queue long after it was put there — the box did not take it, so (for one) an agent update never arrives. Before it,
// a job that ran out left no trace anywhere (clock list CA09-4, CA09-7).
//
// The hub does not open a signed job (it stores and serves bytes), so it cannot read the job's own end time. It
// judges by AGE in the queue: a box asks every 15 minutes at the latest, so a job that waits two hours was not taken.
// One event per job, told once (the hub's time-by-key state); nothing is removed from the queue. The other half — a
// job the box took and REFUSED — is the agent's word at the moment it clears the job (api.handleDeleteJob).
// Pinned by TestSignedJobWait_*.
type SignedJobWaitChecker struct {
pending func() ([]PendingSignedJob, error)
customer func(hostID string) string
told func(key string) time.Time
setTold func(key string, t time.Time) error
onEvent EventNotifyFunc
logger *log.Logger
now func() time.Time
after time.Duration
}
// PendingSignedJob is one job in a box's queue (store.PendingSignedJobs).
type PendingSignedJob struct {
HostID string
JobID string
CreatedAt time.Time
}
const (
// EventSignedJobNotTaken is operator-only (notify.operatorOnlyEvents).
EventSignedJobNotTaken = "signed_job_not_taken"
signedJobWaitsTooLong = 2 * time.Hour
)
func NewSignedJobWaitChecker(pending func() ([]PendingSignedJob, error), customer func(string) string,
told func(string) time.Time, setTold func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger) *SignedJobWaitChecker {
return &SignedJobWaitChecker{pending: pending, customer: customer, told: told, setTold: setTold, onEvent: onEvent,
logger: logger, now: time.Now, after: signedJobWaitsTooLong}
}
// Check runs on the hub's 60 s sweep.
func (c *SignedJobWaitChecker) Check() {
jobs, err := c.pending()
if err != nil {
c.logger.Printf("[WARN] signed-job wait check: cannot read the queue: %v", err)
return
}
now := c.now()
for _, j := range jobs {
if j.CreatedAt.IsZero() || now.Sub(j.CreatedAt) < c.after {
continue
}
key := "signed_job_wait:" + j.HostID + ":" + j.JobID
if !c.told(key).IsZero() {
continue
}
_ = c.setTold(key, now)
waited := now.Sub(j.CreatedAt).Round(time.Minute)
c.logger.Printf("[WARN] signed job %s for host %s was not taken (in the queue for %s)", j.JobID, j.HostID, waited)
msg := fmt.Sprintf("The signed job %s for %s is still in the queue after %s: the box did not take it. A box asks for its jobs every 15 minutes, so the box is off, cannot reach the hub, or its agent does not run. The job's validity has probably run out. Do: read why the box does not report, then sign a new job.",
j.JobID, j.HostID, waited)
details, _ := json.Marshal(map[string]any{"host_id": j.HostID, "job_id": j.JobID,
"queued_at": j.CreatedAt.UTC().Format(time.RFC3339), "waited_minutes": int(waited / time.Minute)})
if c.onEvent != nil {
c.onEvent(c.customer(j.HostID), EventSignedJobNotTaken, "warning", msg, string(details), "hub")
}
}
}