gates / gates (push) Successful in 2m9s
A hub daily check skipped while the hub was down; an operator mail that was never sent; a signed job a box did not take; a signed job a box refused, and the guest-power watchdog's give-up (both from the agent). Operator-only. No new table. Not deployed. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
77 lines
3.2 KiB
Go
77 lines
3.2 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
)
|
|
|
|
// SignedJobWaitChecker (R-1383, D78, `09` §3 decision 299) tells the operator when a signed job is still in a box's
|
|
// queue long after it was put there — the box did not take it, so (for one) an agent update never arrives. Before it,
|
|
// a job that ran out left no trace anywhere (clock list CA09-4, CA09-7).
|
|
//
|
|
// The hub does not open a signed job (it stores and serves bytes), so it cannot read the job's own end time. It
|
|
// judges by AGE in the queue: a box asks every 15 minutes at the latest, so a job that waits two hours was not taken.
|
|
// One event per job, told once (the hub's time-by-key state); nothing is removed from the queue. The other half — a
|
|
// job the box took and REFUSED — is the agent's word at the moment it clears the job (api.handleDeleteJob).
|
|
// Pinned by TestSignedJobWait_*.
|
|
type SignedJobWaitChecker struct {
|
|
pending func() ([]PendingSignedJob, error)
|
|
customer func(hostID string) string
|
|
told func(key string) time.Time
|
|
setTold func(key string, t time.Time) error
|
|
onEvent EventNotifyFunc
|
|
logger *log.Logger
|
|
now func() time.Time
|
|
after time.Duration
|
|
}
|
|
|
|
// PendingSignedJob is one job in a box's queue (store.PendingSignedJobs).
|
|
type PendingSignedJob struct {
|
|
HostID string
|
|
JobID string
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
const (
|
|
// EventSignedJobNotTaken is operator-only (notify.operatorOnlyEvents).
|
|
EventSignedJobNotTaken = "signed_job_not_taken"
|
|
signedJobWaitsTooLong = 2 * time.Hour
|
|
)
|
|
|
|
func NewSignedJobWaitChecker(pending func() ([]PendingSignedJob, error), customer func(string) string,
|
|
told func(string) time.Time, setTold func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger) *SignedJobWaitChecker {
|
|
return &SignedJobWaitChecker{pending: pending, customer: customer, told: told, setTold: setTold, onEvent: onEvent,
|
|
logger: logger, now: time.Now, after: signedJobWaitsTooLong}
|
|
}
|
|
|
|
// Check runs on the hub's 60 s sweep.
|
|
func (c *SignedJobWaitChecker) Check() {
|
|
jobs, err := c.pending()
|
|
if err != nil {
|
|
c.logger.Printf("[WARN] signed-job wait check: cannot read the queue: %v", err)
|
|
return
|
|
}
|
|
now := c.now()
|
|
for _, j := range jobs {
|
|
if j.CreatedAt.IsZero() || now.Sub(j.CreatedAt) < c.after {
|
|
continue
|
|
}
|
|
key := "signed_job_wait:" + j.HostID + ":" + j.JobID
|
|
if !c.told(key).IsZero() {
|
|
continue
|
|
}
|
|
_ = c.setTold(key, now)
|
|
waited := now.Sub(j.CreatedAt).Round(time.Minute)
|
|
c.logger.Printf("[WARN] signed job %s for host %s was not taken (in the queue for %s)", j.JobID, j.HostID, waited)
|
|
msg := fmt.Sprintf("The signed job %s for %s is still in the queue after %s: the box did not take it. A box asks for its jobs every 15 minutes, so the box is off, cannot reach the hub, or its agent does not run. The job's validity has probably run out. Do: read why the box does not report, then sign a new job.",
|
|
j.JobID, j.HostID, waited)
|
|
details, _ := json.Marshal(map[string]any{"host_id": j.HostID, "job_id": j.JobID,
|
|
"queued_at": j.CreatedAt.UTC().Format(time.RFC3339), "waited_minutes": int(waited / time.Minute)})
|
|
if c.onEvent != nil {
|
|
c.onEvent(c.customer(j.HostID), EventSignedJobNotTaken, "warning", msg, string(details), "hub")
|
|
}
|
|
}
|
|
}
|