77 lines
3.3 KiB
Go
77 lines
3.3 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
)
|
|
|
|
// SignedJobWaitChecker (R-1383, D78, `09` §3 decision 299) tells the operator when a signed job is still in a box's
|
|
// queue long after it was put there — the box did not take it, so (for one) an agent update never arrives. Before it,
|
|
// a job that ran out left no trace anywhere (clock list CA09-4, CA09-7).
|
|
//
|
|
// The hub does not open a signed job (it stores and serves bytes), so it cannot read the job's own end time. It
|
|
// judges by AGE in the queue: a box asks every 15 minutes at the latest, so a job that waits two hours was not taken.
|
|
// One event per job, told once (the hub's time-by-key state); nothing is removed from the queue. The other half — a
|
|
// job the box took and REFUSED — is the agent's word at the moment it clears the job (api.handleDeleteJob).
|
|
// Pinned by TestSignedJobWait_*.
|
|
type SignedJobWaitChecker struct {
|
|
pending func() ([]PendingSignedJob, error)
|
|
customer func(hostID string) string
|
|
told func(key string) time.Time
|
|
setTold func(key string, t time.Time) error
|
|
onEvent EventNotifyFunc
|
|
logger *log.Logger
|
|
now func() time.Time
|
|
after time.Duration
|
|
}
|
|
|
|
// PendingSignedJob is one job in a box's queue (store.PendingSignedJobs).
|
|
type PendingSignedJob struct {
|
|
HostID string
|
|
JobID string
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
const (
|
|
// EventSignedJobNotTaken is operator-only (notify.operatorOnlyEvents).
|
|
EventSignedJobNotTaken = "signed_job_not_taken"
|
|
signedJobWaitsTooLong = 2 * time.Hour
|
|
)
|
|
|
|
func NewSignedJobWaitChecker(pending func() ([]PendingSignedJob, error), customer func(string) string,
|
|
told func(string) time.Time, setTold func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger) *SignedJobWaitChecker {
|
|
return &SignedJobWaitChecker{pending: pending, customer: customer, told: told, setTold: setTold, onEvent: onEvent,
|
|
logger: logger, now: time.Now, after: signedJobWaitsTooLong}
|
|
}
|
|
|
|
// Check runs on the hub's 60 s sweep.
|
|
func (c *SignedJobWaitChecker) Check() {
|
|
jobs, err := c.pending()
|
|
if err != nil {
|
|
c.logger.Printf("[WARN] signed-job wait check: cannot read the queue: %v", err)
|
|
return
|
|
}
|
|
now := c.now()
|
|
for _, j := range jobs {
|
|
if j.CreatedAt.IsZero() || now.Sub(j.CreatedAt) < c.after {
|
|
continue
|
|
}
|
|
key := "signed_job_wait:" + j.HostID + ":" + j.JobID
|
|
if !c.told(key).IsZero() {
|
|
continue
|
|
}
|
|
_ = c.setTold(key, now)
|
|
waited := now.Sub(j.CreatedAt).Round(time.Minute)
|
|
c.logger.Printf("[WARN] signed job %s for host %s was not taken (in the queue for %s)", j.JobID, j.HostID, waited)
|
|
msg := fmt.Sprintf("The signed job %s for %s is still in the queue after %s: the box did not take it. A box asks for its jobs every 15 minutes, so one of these is true: the box is off, it cannot reach the hub, its agent does not run — or its agent is too old to know this kind of job and leaves it in the queue on purpose. The job's validity has probably run out. Do: read the box's last report and its agent version, then sign a new job.",
|
|
j.JobID, j.HostID, waited)
|
|
details, _ := json.Marshal(map[string]any{"host_id": j.HostID, "job_id": j.JobID,
|
|
"queued_at": j.CreatedAt.UTC().Format(time.RFC3339), "waited_minutes": int(waited / time.Minute), "subject": j.HostID + ":" + j.JobID})
|
|
if c.onEvent != nil {
|
|
c.onEvent(c.customer(j.HostID), EventSignedJobNotTaken, "warning", msg, string(details), "hub")
|
|
}
|
|
}
|
|
}
|