agent (unreleased): a refused signed job and a watchdog give-up reach the hub (R-1383, D78)
gates / gates (push) Successful in 42s

The job runner clears a job with its outcome, a closed reason code and the op's name. The guest-power watchdog's
give-up rides the host report until the guest runs.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
admin committed 2026-11-14 17:29:53 +01:00
1 parent ede5d0e6c7
commit a1135018c7
11 files changed
+327 -15

No files matched your search

+36
View File
@@ -3,6 +3,7 @@ package localapi
import (
"context"
"log/slog"
"sort"
"sync"
"time"
@@ -82,6 +83,40 @@ type guestPowerState struct {
attempts int
nextAt time.Time
raised bool // the give-up fault has already been raised for this run
raisedAt time.Time
}
// GuestPowerGiveUp is one guest the watchdog stopped trying to start (R-1383, D78). It rides the host report
// (`guest_power_gave_up`) so the operator hears it: until this, the give-up was one ERROR line in the host's journal
// while the household's apps were off. Memory only — an agent restart forgets it (and tries three times again).
type GuestPowerGiveUp struct {
VMID int `json:"vmid"`
Since string `json:"since"` // RFC3339 UTC, when the watchdog gave up
Attempts int `json:"attempts"`
}
// GuestPowerGaveUpStanza is GuestPowerGaveUp shaped for the host report: an UNTYPED nil when there is none, so the
// report omits the field (a typed nil slice in an `any` would be sent as `null`).
func (s *Server) GuestPowerGaveUpStanza() any {
if g := s.GuestPowerGaveUp(); len(g) > 0 {
return g
}
return nil
}
// GuestPowerGaveUp returns the guests the watchdog has given up on, by vmid. nil when there is none — the report
// stanza is then omitted. Pinned by TestGuestPower_GiveUpIsReportedUntilTheGuestRuns.
func (s *Server) GuestPowerGaveUp() []GuestPowerGiveUp {
guestPowerMu.Lock()
defer guestPowerMu.Unlock()
var out []GuestPowerGiveUp
for vmid, st := range s.guestPower {
if st.raised {
out = append(out, GuestPowerGiveUp{VMID: vmid, Since: st.raisedAt.UTC().Format(time.RFC3339), Attempts: st.attempts})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].VMID < out[j].VMID })
return out
}
// WatchGuestPower runs the guest-power sweep every guestPowerInterval until ctx is done. No-op when
@@ -260,6 +295,7 @@ func (s *Server) markGuestPowerRaised(vmid int) {
defer guestPowerMu.Unlock()
st := s.guestPower[vmid]
st.raised = true
st.raisedAt = s.now()
s.guestPower[vmid] = st
}
+44
View File
@@ -2,6 +2,7 @@ package localapi
import (
"context"
"encoding/json"
"errors"
"log/slog"
"testing"
@@ -237,3 +238,46 @@ func TestGuestPower_RunningGuestClearsAttemptHistory(t *testing.T) {
t.Error("attempt history survived the guest coming back healthy")
}
}
// R-1383 (D78): the give-up was ONE journal line while the household's apps were off. It now rides the host report:
// nothing before the give-up (and an untyped nil, so the report omits the field), the guest with the time and the
// attempt count after it, and nothing again once the guest runs.
// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/B/redproof-agent-guest-power.txt).
func TestGuestPower_GiveUpIsReportedUntilTheGuestRuns(t *testing.T) {
ctl := &fakeGuestPowerCtl{
guests: []proxmox.Guest{{VMID: 9201, Status: "stopped"}},
locks: map[int]string{9201: ""},
onboot: map[int]bool{9201: true},
startErr: errors.New("cannot start: storage offline"),
}
now := time.Date(2026, 11, 14, 2, 0, 0, 0, time.UTC)
s := gpServer(t, ctl, func() time.Time { return now })
if st := s.GuestPowerGaveUpStanza(); st != nil {
t.Fatalf("before any attempt the stanza is %#v, want an untyped nil (the report omits the field)", st)
}
for i := 0; i < guestPowerMaxAttempts; i++ { // the three failed starts
s.GuestPowerTick(context.Background())
if g := s.GuestPowerGaveUp(); len(g) != 0 {
t.Fatalf("after %d failed start(s) the watchdog still tries, but it reports a give-up: %+v", i+1, g)
}
now = now.Add(10 * time.Minute)
}
gaveUpAt := now
s.GuestPowerTick(context.Background()) // the tick that gives up
now = now.Add(time.Hour)
s.GuestPowerTick(context.Background())
g := s.GuestPowerGaveUp()
if len(g) != 1 || g[0].VMID != 9201 || g[0].Attempts != guestPowerMaxAttempts || g[0].Since != gaveUpAt.Format(time.RFC3339) {
t.Fatalf("give-up list = %+v, want guest 9201, %d attempts, since %s (the give-up's time, not a later tick's)", g, guestPowerMaxAttempts, gaveUpAt.Format(time.RFC3339))
}
b, _ := json.Marshal(s.GuestPowerGaveUpStanza())
if string(b) != `[{"vmid":9201,"since":"2026-11-14T02:30:00Z","attempts":3}]` {
t.Fatalf("the stanza on the wire = %s", b)
}
// the operator starts the guest: the next sweep sees it running and the list is empty again
ctl.guests = []proxmox.Guest{{VMID: 9201, Status: "running"}}
s.GuestPowerTick(context.Background())
if st := s.GuestPowerGaveUpStanza(); st != nil {
t.Fatalf("the guest runs again and the stanza is still %#v", st)
}
}