agent (unreleased): a refused signed job and a watchdog give-up reach the hub (R-1383, D78)
gates / gates (push) Successful in 42s
gates / gates (push) Successful in 42s
The job runner clears a job with its outcome, a closed reason code and the op's name. The guest-power watchdog's give-up rides the host report until the guest runs. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
ede5d0e6c7
commit
a1135018c7
11 files changed
+327
-15
No files matched your search
@@ -3,6 +3,7 @@ package localapi
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -82,6 +83,40 @@ type guestPowerState struct {
|
||||
attempts int
|
||||
nextAt time.Time
|
||||
raised bool // the give-up fault has already been raised for this run
|
||||
raisedAt time.Time
|
||||
}
|
||||
|
||||
// GuestPowerGiveUp is one guest the watchdog stopped trying to start (R-1383, D78). It rides the host report
|
||||
// (`guest_power_gave_up`) so the operator hears it: until this, the give-up was one ERROR line in the host's journal
|
||||
// while the household's apps were off. Memory only — an agent restart forgets it (and tries three times again).
|
||||
type GuestPowerGiveUp struct {
|
||||
VMID int `json:"vmid"`
|
||||
Since string `json:"since"` // RFC3339 UTC, when the watchdog gave up
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
// GuestPowerGaveUpStanza is GuestPowerGaveUp shaped for the host report: an UNTYPED nil when there is none, so the
|
||||
// report omits the field (a typed nil slice in an `any` would be sent as `null`).
|
||||
func (s *Server) GuestPowerGaveUpStanza() any {
|
||||
if g := s.GuestPowerGaveUp(); len(g) > 0 {
|
||||
return g
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GuestPowerGaveUp returns the guests the watchdog has given up on, by vmid. nil when there is none — the report
|
||||
// stanza is then omitted. Pinned by TestGuestPower_GiveUpIsReportedUntilTheGuestRuns.
|
||||
func (s *Server) GuestPowerGaveUp() []GuestPowerGiveUp {
|
||||
guestPowerMu.Lock()
|
||||
defer guestPowerMu.Unlock()
|
||||
var out []GuestPowerGiveUp
|
||||
for vmid, st := range s.guestPower {
|
||||
if st.raised {
|
||||
out = append(out, GuestPowerGiveUp{VMID: vmid, Since: st.raisedAt.UTC().Format(time.RFC3339), Attempts: st.attempts})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].VMID < out[j].VMID })
|
||||
return out
|
||||
}
|
||||
|
||||
// WatchGuestPower runs the guest-power sweep every guestPowerInterval until ctx is done. No-op when
|
||||
@@ -260,6 +295,7 @@ func (s *Server) markGuestPowerRaised(vmid int) {
|
||||
defer guestPowerMu.Unlock()
|
||||
st := s.guestPower[vmid]
|
||||
st.raised = true
|
||||
st.raisedAt = s.now()
|
||||
s.guestPower[vmid] = st
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"testing"
|
||||
@@ -237,3 +238,46 @@ func TestGuestPower_RunningGuestClearsAttemptHistory(t *testing.T) {
|
||||
t.Error("attempt history survived the guest coming back healthy")
|
||||
}
|
||||
}
|
||||
|
||||
// R-1383 (D78): the give-up was ONE journal line while the household's apps were off. It now rides the host report:
|
||||
// nothing before the give-up (and an untyped nil, so the report omits the field), the guest with the time and the
|
||||
// attempt count after it, and nothing again once the guest runs.
|
||||
// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/B/redproof-agent-guest-power.txt).
|
||||
func TestGuestPower_GiveUpIsReportedUntilTheGuestRuns(t *testing.T) {
|
||||
ctl := &fakeGuestPowerCtl{
|
||||
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
|
||||
locks: map[int]string{9201: ""},
|
||||
onboot: map[int]bool{9201: true},
|
||||
startErr: errors.New("cannot start: storage offline"),
|
||||
}
|
||||
now := time.Date(2026, 11, 14, 2, 0, 0, 0, time.UTC)
|
||||
s := gpServer(t, ctl, func() time.Time { return now })
|
||||
if st := s.GuestPowerGaveUpStanza(); st != nil {
|
||||
t.Fatalf("before any attempt the stanza is %#v, want an untyped nil (the report omits the field)", st)
|
||||
}
|
||||
for i := 0; i < guestPowerMaxAttempts; i++ { // the three failed starts
|
||||
s.GuestPowerTick(context.Background())
|
||||
if g := s.GuestPowerGaveUp(); len(g) != 0 {
|
||||
t.Fatalf("after %d failed start(s) the watchdog still tries, but it reports a give-up: %+v", i+1, g)
|
||||
}
|
||||
now = now.Add(10 * time.Minute)
|
||||
}
|
||||
gaveUpAt := now
|
||||
s.GuestPowerTick(context.Background()) // the tick that gives up
|
||||
now = now.Add(time.Hour)
|
||||
s.GuestPowerTick(context.Background())
|
||||
g := s.GuestPowerGaveUp()
|
||||
if len(g) != 1 || g[0].VMID != 9201 || g[0].Attempts != guestPowerMaxAttempts || g[0].Since != gaveUpAt.Format(time.RFC3339) {
|
||||
t.Fatalf("give-up list = %+v, want guest 9201, %d attempts, since %s (the give-up's time, not a later tick's)", g, guestPowerMaxAttempts, gaveUpAt.Format(time.RFC3339))
|
||||
}
|
||||
b, _ := json.Marshal(s.GuestPowerGaveUpStanza())
|
||||
if string(b) != `[{"vmid":9201,"since":"2026-11-14T02:30:00Z","attempts":3}]` {
|
||||
t.Fatalf("the stanza on the wire = %s", b)
|
||||
}
|
||||
// the operator starts the guest: the next sweep sees it running and the list is empty again
|
||||
ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}}
|
||||
s.GuestPowerTick(context.Background())
|
||||
if st := s.GuestPowerGaveUpStanza(); st != nil {
|
||||
t.Fatalf("the guest runs again and the stanza is still %#v", st)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user