hub: box presence from the wait channel + delete a switched-off box at once (R-30, D2)
Slice 1: intent.Hub records one start/end per GET /api/v1/wait request and answers Presence(customer, now): connected (hold open, or one started < 333 s = 243 s cadence + 90 s grace ago), not connected since T, or unknown (hub up < 333 s; in memory only). The host page shows "Box connection". One DEBUG line per presence change. Slice 2 (operator ruling D2, 2026-10-08, 09 §3 decision 186): a host that is online by its report clock but whose box has had no wait-channel connection for >= 360 s may be deleted at once after the tick "I checked: the box is off". Presence is re-read at POST time; the tick alone, unknown presence, a connected box or a shorter gap keep today's 409. The delete logs the operator channel and saves one host_deleted_box_off event. RESET and the customer-delete cascade are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -604,6 +604,7 @@ func main() {
|
||||
// compares with != , so a restart costs exactly one harmless full-state report. Closed before
|
||||
// server.Shutdown (below) so held waits complete instantly instead of eating the grace window.
|
||||
intentHub := intent.New()
|
||||
intentHub.SetLogger(logger.Printf) // R-30: one DEBUG line per box-presence change
|
||||
apiHandler.SetIntentHub(intentHub)
|
||||
webServer.SetIntentHub(intentHub)
|
||||
|
||||
|
||||
@@ -48,6 +48,11 @@ func (h *Handler) handleWait(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// R-30: box presence — recorded ONCE per request (intent.Wait below runs once per heartbeat
|
||||
// window), and ended on every exit path.
|
||||
h.intentHub.MarkWaitStart(customerID)
|
||||
defer h.intentHub.MarkWaitEnd(customerID)
|
||||
|
||||
var lastSeen uint64
|
||||
if s := r.URL.Query().Get("gen"); s != "" {
|
||||
if v, err := strconv.ParseUint(s, 10, 64); err == nil {
|
||||
|
||||
@@ -181,3 +181,30 @@ func TestWait_CustomerIsolation(t *testing.T) {
|
||||
t.Fatalf("A should time out on its own gen 0 (isolation); body = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// R-30 slice 1: each wait REQUEST marks one start and one end on the intent hub's presence — open
|
||||
// while held, ended on return. Without the marks the box reads "never seen"; without the end mark it
|
||||
// would read connected forever.
|
||||
func TestWait_MarksBoxPresence(t *testing.T) {
|
||||
h, st, hub := newWaitHandler(t)
|
||||
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c", RetrievalPassword: "pw", APIKey: "CKEY", ConfigJSON: "{}"}); err != nil {
|
||||
t.Fatalf("SaveCustomerConfig: %v", err)
|
||||
}
|
||||
t0 := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
clock := t0.Add(-time.Hour)
|
||||
hub.SetClock(func() time.Time { return clock }) // born an hour before t0
|
||||
clock = t0
|
||||
shrinkWaitTiming(t, 2*time.Second, 500*time.Millisecond)
|
||||
|
||||
ch := doWaitAsync(h, "CKEY", "0")
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if p := hub.Presence("c", t0.Add(time.Hour)); p.State != intent.PresenceConnected {
|
||||
t.Fatalf("while the wait is held: %+v, want connected (open hold)", p)
|
||||
}
|
||||
hub.Bump("c")
|
||||
recvRR(t, ch, time.Second)
|
||||
p := hub.Presence("c", t0.Add(intent.PresenceConnectedWindow+time.Second))
|
||||
if p.State != intent.PresenceNotConnected || p.NeverSeen || !p.Since.Equal(t0) {
|
||||
t.Fatalf("after the wait returned: %+v, want not_connected since t0 (start and end both marked)", p)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,14 +32,24 @@ type Hub struct {
|
||||
waiters map[string][]chan struct{} // customerID -> registered wake channels (buffered, cap 1)
|
||||
closed chan struct{} // closed by Close() to wake every waiter
|
||||
isClosed bool
|
||||
|
||||
// R-30 box presence (presence.go): per-customer wait-request bookkeeping, the hub's own start
|
||||
// time (a hub younger than PresenceConnectedWindow answers "unknown"), a clock seam and an
|
||||
// optional debug logger.
|
||||
presence map[string]*presenceRec
|
||||
born time.Time
|
||||
now func() time.Time
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// New builds an empty intent hub.
|
||||
func New() *Hub {
|
||||
return &Hub{
|
||||
gen: make(map[string]uint64),
|
||||
waiters: make(map[string][]chan struct{}),
|
||||
closed: make(chan struct{}),
|
||||
gen: make(map[string]uint64),
|
||||
waiters: make(map[string][]chan struct{}),
|
||||
closed: make(chan struct{}),
|
||||
presence: make(map[string]*presenceRec),
|
||||
born: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Box presence from the wait channel (R-30, design `audits/day-2026-10-08/design-R-30.md` option B).
|
||||
//
|
||||
// A healthy controller holds GET /api/v1/wait for waitMaxHold (240 s) and starts the next hold at
|
||||
// once; measured on the ingress 2026-10-08: one new wait per box every 241–243 s. So "a hold is open,
|
||||
// or one STARTED less than PresenceConnectedWindow ago" means the box's controller is running and
|
||||
// reaching the hub. A box that loses power ends its hold normally (the hub ends it at 240 s) and then
|
||||
// no new wait arrives — that gap is the signal.
|
||||
//
|
||||
// IN MEMORY ONLY, like the generation counter. A hub that started less than PresenceConnectedWindow
|
||||
// ago has not had the chance to see a healthy box's next wait, so it answers PresenceUnknown — never
|
||||
// PresenceNotConnected. Callers fall back to the report clock on Unknown (the safe direction).
|
||||
// Pinned by presence_test.go TestPresence_HubRestartIsUnknown.
|
||||
const (
|
||||
// PresenceWaitCadence is the measured interval between two wait starts from one healthy box.
|
||||
PresenceWaitCadence = 243 * time.Second
|
||||
// PresenceGrace absorbs a slow reconnect (TLS, DNS, a busy box) on top of the cadence.
|
||||
PresenceGrace = 90 * time.Second
|
||||
// PresenceConnectedWindow = cadence + grace (333 s): a wait started within it means "connected".
|
||||
PresenceConnectedWindow = PresenceWaitCadence + PresenceGrace
|
||||
)
|
||||
|
||||
// Presence states.
|
||||
const (
|
||||
PresenceConnected = "connected"
|
||||
PresenceNotConnected = "not_connected"
|
||||
PresenceUnknown = "unknown"
|
||||
)
|
||||
|
||||
// Presence is one customer's box-connection verdict at a moment.
|
||||
type Presence struct {
|
||||
State string
|
||||
// Since is the start of the not-connected span (the last hold's end, or the hub's own start when
|
||||
// no wait was seen since). Zero unless State == PresenceNotConnected.
|
||||
Since time.Time
|
||||
// NeverSeen: no wait from this customer since the hub started (Since is then the hub's start).
|
||||
NeverSeen bool
|
||||
}
|
||||
|
||||
// NotConnectedFor is how long the box has been without a connection at now (0 unless not connected).
|
||||
func (p Presence) NotConnectedFor(now time.Time) time.Duration {
|
||||
if p.State != PresenceNotConnected {
|
||||
return 0
|
||||
}
|
||||
if d := now.Sub(p.Since); d > 0 {
|
||||
return d
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type presenceRec struct {
|
||||
open int
|
||||
lastStart time.Time
|
||||
lastEnd time.Time
|
||||
lastState string // for the debug line on a change
|
||||
}
|
||||
|
||||
func (h *Hub) clock() time.Time {
|
||||
if h.now != nil {
|
||||
return h.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// SetClock replaces the presence clock (tests). Must be called before use; also resets the hub's
|
||||
// start time to the new clock's now.
|
||||
func (h *Hub) SetClock(now func() time.Time) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.now = now
|
||||
h.born = h.clock()
|
||||
}
|
||||
|
||||
// SetLogger wires a Printf-style logger for the presence debug lines. Nil = silent.
|
||||
func (h *Hub) SetLogger(logf func(format string, args ...any)) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.logf = logf
|
||||
}
|
||||
|
||||
func (h *Hub) rec(customerID string) *presenceRec {
|
||||
r := h.presence[customerID]
|
||||
if r == nil {
|
||||
r = &presenceRec{}
|
||||
h.presence[customerID] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// MarkWaitStart records that a wait request from the customer's box began. Called ONCE per HTTP
|
||||
// request by the wait handler (not inside Wait, which runs once per heartbeat window).
|
||||
func (h *Hub) MarkWaitStart(customerID string) {
|
||||
if customerID == "" {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.clock()
|
||||
r := h.rec(customerID)
|
||||
r.open++
|
||||
r.lastStart = now
|
||||
h.noteStateLocked(customerID, r, now)
|
||||
}
|
||||
|
||||
// MarkWaitEnd records that a wait request ended (any exit path).
|
||||
func (h *Hub) MarkWaitEnd(customerID string) {
|
||||
if customerID == "" {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r := h.rec(customerID)
|
||||
if r.open > 0 {
|
||||
r.open--
|
||||
}
|
||||
r.lastEnd = h.clock()
|
||||
}
|
||||
|
||||
// Presence answers the customer's box-connection verdict at now.
|
||||
func (h *Hub) Presence(customerID string, now time.Time) Presence {
|
||||
if customerID == "" {
|
||||
return Presence{State: PresenceUnknown}
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r := h.presence[customerID]
|
||||
p := h.presenceLocked(r, now)
|
||||
if r != nil {
|
||||
h.noteStateLocked(customerID, r, now)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (h *Hub) presenceLocked(r *presenceRec, now time.Time) Presence {
|
||||
if r != nil {
|
||||
if r.open > 0 || (!r.lastStart.IsZero() && now.Sub(r.lastStart) < PresenceConnectedWindow) {
|
||||
return Presence{State: PresenceConnected}
|
||||
}
|
||||
}
|
||||
// The hub itself has not been up long enough to have seen a healthy box's next wait.
|
||||
if now.Sub(h.born) < PresenceConnectedWindow {
|
||||
return Presence{State: PresenceUnknown}
|
||||
}
|
||||
if r == nil || r.lastStart.IsZero() {
|
||||
return Presence{State: PresenceNotConnected, Since: h.born, NeverSeen: true}
|
||||
}
|
||||
since := r.lastEnd
|
||||
if since.Before(r.lastStart) {
|
||||
since = r.lastStart
|
||||
}
|
||||
return Presence{State: PresenceNotConnected, Since: since}
|
||||
}
|
||||
|
||||
// noteStateLocked writes one DEBUG line when a customer's presence verdict changes.
|
||||
func (h *Hub) noteStateLocked(customerID string, r *presenceRec, now time.Time) {
|
||||
st := h.presenceLocked(r, now).State
|
||||
if st == r.lastState {
|
||||
return
|
||||
}
|
||||
prev := r.lastState
|
||||
r.lastState = st
|
||||
if h.logf != nil && prev != "" {
|
||||
h.logf("[DEBUG] box presence %s: %s -> %s (open holds %d)", customerID, prev, st, r.open)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeClock is a settable clock for the presence seam.
|
||||
type fakeClock struct{ t time.Time }
|
||||
|
||||
func (c *fakeClock) now() time.Time { return c.t }
|
||||
|
||||
// A hub that has been up long enough to judge (born well before t0).
|
||||
func judgingHub(t0 time.Time) (*Hub, *fakeClock) {
|
||||
c := &fakeClock{t: t0.Add(-time.Hour)}
|
||||
h := New()
|
||||
h.SetClock(c.now)
|
||||
c.t = t0
|
||||
return h, c
|
||||
}
|
||||
|
||||
// R-30 §4 test 1: a wait starts at t0 and ends at t0+240 s, and none follows → at t0+300 s the box is
|
||||
// connected (a wait started < 333 s ago), at t0+334 s it is "not connected since t0+240 s".
|
||||
func TestPresence_WindowsAfterOneHold(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
h, c := judgingHub(t0)
|
||||
h.MarkWaitStart("c1")
|
||||
if p := h.Presence("c1", t0.Add(100*time.Second)); p.State != PresenceConnected {
|
||||
t.Fatalf("during the hold: %+v, want connected", p)
|
||||
}
|
||||
c.t = t0.Add(240 * time.Second)
|
||||
h.MarkWaitEnd("c1")
|
||||
if p := h.Presence("c1", t0.Add(300*time.Second)); p.State != PresenceConnected {
|
||||
t.Fatalf("t0+300 s: %+v, want connected (started < 333 s ago)", p)
|
||||
}
|
||||
p := h.Presence("c1", t0.Add(334*time.Second))
|
||||
if p.State != PresenceNotConnected || !p.Since.Equal(t0.Add(240*time.Second)) || p.NeverSeen {
|
||||
t.Fatalf("t0+334 s: %+v, want not_connected since t0+240 s", p)
|
||||
}
|
||||
if got := p.NotConnectedFor(t0.Add(600 * time.Second)); got != 360*time.Second {
|
||||
t.Fatalf("NotConnectedFor at t0+600 s = %s, want 6m0s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// R-30 §4 test 2: a new start at t0+242 s keeps the box connected well past t0+334 s.
|
||||
func TestPresence_NextHoldKeepsConnected(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
h, c := judgingHub(t0)
|
||||
h.MarkWaitStart("c1")
|
||||
c.t = t0.Add(240 * time.Second)
|
||||
h.MarkWaitEnd("c1")
|
||||
c.t = t0.Add(242 * time.Second)
|
||||
h.MarkWaitStart("c1")
|
||||
for _, at := range []time.Duration{334 * time.Second, 500 * time.Second, 574 * time.Second} {
|
||||
if p := h.Presence("c1", t0.Add(at)); p.State != PresenceConnected {
|
||||
t.Fatalf("t0+%s: %+v, want connected (a hold is open)", at, p)
|
||||
}
|
||||
}
|
||||
// Two overlapping requests: one ending does not close the other.
|
||||
h.MarkWaitStart("c1")
|
||||
h.MarkWaitEnd("c1")
|
||||
if p := h.Presence("c1", t0.Add(time.Hour)); p.State != PresenceConnected {
|
||||
t.Fatalf("one of two holds still open: %+v, want connected", p)
|
||||
}
|
||||
}
|
||||
|
||||
// R-30 §4 test 3: a new Hub (a hub restart) answers unknown — never not_connected — until it has
|
||||
// been up for the connected window; after that, a customer never seen reads not_connected since the
|
||||
// hub's start, flagged NeverSeen.
|
||||
func TestPresence_HubRestartIsUnknown(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
c := &fakeClock{t: t0}
|
||||
h := New()
|
||||
h.SetClock(c.now)
|
||||
for _, at := range []time.Duration{0, 100 * time.Second, PresenceConnectedWindow - time.Second} {
|
||||
if p := h.Presence("c1", t0.Add(at)); p.State != PresenceUnknown {
|
||||
t.Fatalf("hub up %s: %+v, want unknown", at, p)
|
||||
}
|
||||
}
|
||||
p := h.Presence("c1", t0.Add(PresenceConnectedWindow))
|
||||
if p.State != PresenceNotConnected || !p.NeverSeen || !p.Since.Equal(t0) {
|
||||
t.Fatalf("hub up 333 s, box never seen: %+v, want not_connected since hub start", p)
|
||||
}
|
||||
if p := h.Presence("", t0.Add(time.Hour)); p.State != PresenceUnknown {
|
||||
t.Fatalf("no customer: %+v, want unknown", p)
|
||||
}
|
||||
}
|
||||
|
||||
// The window is the measured cadence plus the grace (pins the constants the delete guard leans on).
|
||||
func TestPresence_WindowConstants(t *testing.T) {
|
||||
if PresenceConnectedWindow != 333*time.Second {
|
||||
t.Fatalf("PresenceConnectedWindow = %s, want 5m33s (243 s + 90 s)", PresenceConnectedWindow)
|
||||
}
|
||||
}
|
||||
|
||||
// One DEBUG line per change, none on a repeat read.
|
||||
func TestPresence_DebugLineOnChange(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
h, c := judgingHub(t0)
|
||||
var lines []string
|
||||
h.SetLogger(func(f string, a ...any) { lines = append(lines, f) })
|
||||
h.MarkWaitStart("c1")
|
||||
c.t = t0.Add(240 * time.Second)
|
||||
h.MarkWaitEnd("c1")
|
||||
h.Presence("c1", t0.Add(400*time.Second))
|
||||
h.Presence("c1", t0.Add(500*time.Second))
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "[DEBUG] box presence") {
|
||||
t.Fatalf("debug lines = %q, want exactly one change line", lines)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/intent"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
|
||||
@@ -712,8 +713,54 @@ func parseReportedAgentBinary(reportJSON string) (version, sha string) {
|
||||
return strings.TrimSpace(doc.AgentVersion), strings.ToLower(strings.TrimSpace(doc.AgentSHA256))
|
||||
}
|
||||
|
||||
// hostOffDeleteAfter is how long the hub must have had no wait-channel connection from a box before
|
||||
// "delete host" may go ahead at once on the operator's tick "I checked: the box is off" (R-30 slice 2,
|
||||
// operator ruling D2 2026-10-08, `09` §3 decision 186). It is deliberately LONGER than
|
||||
// intent.PresenceConnectedWindow (333 s): a not-connected span of 360 s after the last hold ended
|
||||
// means at least one full expected reconnect (243 s cadence + 90 s grace) was missed, and a hub that
|
||||
// restarted less than 333 s ago answers "unknown", which never permits this path.
|
||||
// Pinned by r30_presence_delete_test.go (TestR30_TickButTooSoonRefused, TestR30_OffDeleteLimitAboveWindow).
|
||||
const hostOffDeleteAfter = 360 * time.Second
|
||||
|
||||
// boxConnectionView is the host page's "Box connection" line (R-30 slice 1).
|
||||
type boxConnectionView struct {
|
||||
State string // intent.Presence* state
|
||||
Since time.Time
|
||||
NeverSeen bool
|
||||
}
|
||||
|
||||
// hostPresence reads the wait-channel presence of the host's customer. No customer, or no intent hub
|
||||
// wired → unknown (the report clock alone decides, as before R-30).
|
||||
func (s *Server) hostPresence(host *store.Host, now time.Time) intent.Presence {
|
||||
if s.intentHub == nil || host == nil || host.CustomerID == "" {
|
||||
return intent.Presence{State: intent.PresenceUnknown}
|
||||
}
|
||||
return s.intentHub.Presence(host.CustomerID, now)
|
||||
}
|
||||
|
||||
// offDeleteOpen: the host reads ONLINE by its report clock, but the hub has had no connection from
|
||||
// its box for at least hostOffDeleteAfter. Only then may the operator's tick delete it at once.
|
||||
func offDeleteOpen(reportStatus string, p intent.Presence, now time.Time) bool {
|
||||
return reportStatus == "ok" && p.State == intent.PresenceNotConnected && p.NotConnectedFor(now) >= hostOffDeleteAfter
|
||||
}
|
||||
|
||||
// operatorChannel names how the operator reached the hub, for the audit line (the hub has one
|
||||
// operator password and no user names).
|
||||
func operatorChannel(r *http.Request) string {
|
||||
if _, err := r.Cookie(SessionCookieName); err == nil {
|
||||
return "operator (browser session)"
|
||||
}
|
||||
if _, _, ok := r.BasicAuth(); ok {
|
||||
return "operator (CLI, Basic auth)"
|
||||
}
|
||||
return "operator"
|
||||
}
|
||||
|
||||
func (s *Server) hostDetailData(host *store.Host, r *http.Request) map[string]interface{} {
|
||||
status := s.hostStatus(host.LastReportAt)
|
||||
now := time.Now()
|
||||
presence := s.hostPresence(host, now)
|
||||
offOpen := offDeleteOpen(status, presence, now)
|
||||
|
||||
guests, _ := s.store.ListGuestsForHost(host.HostID)
|
||||
guestRunning := 0
|
||||
@@ -820,7 +867,13 @@ func (s *Server) hostDetailData(host *store.Host, r *http.Request) map[string]in
|
||||
"CSRFToken": s.getCSRFToken(r),
|
||||
// v0.47.0 stale host removal: the danger-zone card renders ONLY for non-online
|
||||
// hosts — an ONLINE host is never deletable (no override exists).
|
||||
"Deletable": status != "ok",
|
||||
// R-30 slice 2 (D2): an online-by-report host whose box has been unreachable for
|
||||
// hostOffDeleteAfter is deletable too — but only with the "I checked: the box is off" tick,
|
||||
// which handleHostDelete re-checks against presence at POST time.
|
||||
"Deletable": status != "ok" || offOpen,
|
||||
"OffTickRequired": offOpen,
|
||||
// R-30 slice 1: the box's wait-channel presence, beside the report clock.
|
||||
"BoxConnection": boxConnectionView{State: presence.State, Since: presence.Since, NeverSeen: presence.NeverSeen},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -845,10 +898,13 @@ func (s *Server) handleHostDeleteImpact(w http.ResponseWriter, r *http.Request,
|
||||
return
|
||||
}
|
||||
status := s.hostStatus(host.LastReportAt)
|
||||
now := time.Now()
|
||||
offOpen := offDeleteOpen(status, s.hostPresence(host, now), now)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"status": status,
|
||||
"deletable": status != "ok",
|
||||
"deletable": status != "ok" || offOpen,
|
||||
"off_tick_required": offOpen,
|
||||
"guests": a.Guests,
|
||||
"reports": a.Reports,
|
||||
"log_bundles": a.LogBundles,
|
||||
@@ -916,9 +972,11 @@ func (s *Server) handleHostRevealRecoveryCredential(w http.ResponseWriter, r *ht
|
||||
|
||||
// handleHostDelete — POST /hosts/{id}/delete (v0.47.0 stale host removal). Gates, in order:
|
||||
// - unknown host → 404
|
||||
// - ONLINE host → 409 unconditionally (host reports authenticate via GetHostByAPIKey;
|
||||
// deleting a live host permanently bricks its heartbeat channel — enroll is
|
||||
// passphrase-gated mint-once, so there is deliberately NO override)
|
||||
// - ONLINE host → 409 (host reports authenticate via GetHostByAPIKey; deleting a live host
|
||||
// permanently bricks its heartbeat channel — enroll is passphrase-gated mint-once). The ONE
|
||||
// exception (R-30 slice 2, operator ruling D2 2026-10-08): box_off_confirmed=1 AND the hub has
|
||||
// had no wait-channel connection from the box for hostOffDeleteAfter, re-read here at POST time.
|
||||
// Presence "unknown" (hub restarted, no customer, no intent hub) never opens it.
|
||||
// - confirm_host_id mismatch → 400 (type-to-confirm)
|
||||
// - escrow present without delete_escrow=1 → 409 (store-enforced, fail-safe-to-refuse)
|
||||
func (s *Server) handleHostDelete(w http.ResponseWriter, r *http.Request, hostID string) {
|
||||
@@ -932,10 +990,24 @@ func (s *Server) handleHostDelete(w http.ResponseWriter, r *http.Request, hostID
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
boxOffPath := false
|
||||
if status := s.hostStatus(host.LastReportAt); status == "ok" {
|
||||
s.logger.Printf("[WARN] host delete refused: %s is online", hostID)
|
||||
http.Error(w, "Host is ONLINE — deletion is refused (a live agent would receive 401s permanently)."+s.deletionOpensAt(host.LastReportAt, time.Now()), http.StatusConflict)
|
||||
return
|
||||
// R-30 slice 2 (D2): the tick alone is never enough — presence is re-read HERE, at POST time.
|
||||
now := time.Now()
|
||||
presence := s.hostPresence(host, now)
|
||||
ticked := r.FormValue("box_off_confirmed") == "1"
|
||||
if !(ticked && offDeleteOpen(status, presence, now)) {
|
||||
s.logger.Printf("[WARN] host delete refused: %s is online (box connection %s, not connected for %s, tick %t)",
|
||||
hostID, presence.State, presence.NotConnectedFor(now).Round(time.Second), ticked)
|
||||
msg := "Host is ONLINE — deletion is refused (a live agent would receive 401s permanently)." + s.deletionOpensAt(host.LastReportAt, now)
|
||||
if offDeleteOpen(status, presence, now) {
|
||||
msg += " The hub has had no connection from this box for " + presence.NotConnectedFor(now).Round(time.Second).String() +
|
||||
": tick \"I checked: the box is off\" to delete it now."
|
||||
}
|
||||
http.Error(w, msg, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
boxOffPath = true
|
||||
}
|
||||
if confirm := strings.TrimSpace(r.FormValue("confirm_host_id")); confirm != hostID {
|
||||
s.logger.Printf("[WARN] host delete refused: %s confirm mismatch", hostID)
|
||||
@@ -959,6 +1031,17 @@ func (s *Server) handleHostDelete(w http.ResponseWriter, r *http.Request, hostID
|
||||
// delete does, R-600), so an uninstalled box's tunnel stops being accepted by the off-site endpoint
|
||||
// at once instead of on the reconciler's next 5-minute tick.
|
||||
s.logger.Printf("[INFO] host deleted: %s (%s; %s)", hostID, hostDeleteEscrowEffect(escrowBefore != nil, escErr), s.requestWGPeerPush())
|
||||
if boxOffPath {
|
||||
// R-30 slice 2: who pressed it, and one operator event on the customer's timeline.
|
||||
s.logger.Printf("[INFO] host %s deleted while online by its report clock: %s ticked \"I checked: the box is off\" (no box connection for >= %s)",
|
||||
hostID, operatorChannel(r), hostOffDeleteAfter)
|
||||
if host.CustomerID != "" {
|
||||
if _, err := s.store.SaveEvent(host.CustomerID, hostDeletedBoxOffEvent, "warning",
|
||||
"Host "+hostID+" deleted before its report went stale: the operator confirmed the box is off (no box connection for at least "+hostOffDeleteAfter.String()+").", "", "hub"); err != nil {
|
||||
s.logger.Printf("[WARN] SaveEvent %s %s/%s: %v", hostDeletedBoxOffEvent, host.CustomerID, hostID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// R-509: the customer record stays and now waits for a box → send the connect link.
|
||||
if host.CustomerID != "" {
|
||||
s.autoMintSelfBindIfWaiting(host.CustomerID, "host delete")
|
||||
@@ -966,6 +1049,9 @@ func (s *Server) handleHostDelete(w http.ResponseWriter, r *http.Request, hostID
|
||||
http.Redirect(w, r, "/hosts", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// hostDeletedBoxOffEvent is the hub-minted operator event for a delete on the "box is off" tick (R-30).
|
||||
const hostDeletedBoxOffEvent = "host_deleted_box_off"
|
||||
|
||||
// hostDeleteEscrowEffect states what a host delete did to the key escrow, in operator words (R-544).
|
||||
// A host delete NEVER destroys escrow: the store demotes it to retained custody, and only the customer
|
||||
// delete purges it. The old line printed the form flag ("escrow deleted: true"), which read as a
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/intent"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
)
|
||||
|
||||
// R-30 (operator ruling D2, 2026-10-08, `09` §3 decision 186): "delete host" goes ahead at once
|
||||
// after the operator's tick "I checked: the box is off" ONLY when the host is online by its report
|
||||
// clock AND the hub has had no wait-channel connection from its box for hostOffDeleteAfter. Every
|
||||
// other combination keeps today's refusal.
|
||||
|
||||
type r30Presence int
|
||||
|
||||
const (
|
||||
r30Unknown r30Presence = iota // hub "restarted" just now
|
||||
r30Connected // a hold is open
|
||||
r30OffLong // last hold ended 11 min ago
|
||||
r30OffShort // last hold ended 5 min ago (started 8 min ago)
|
||||
r30NeverSeenLong // hub up 20 min, box never waited
|
||||
)
|
||||
|
||||
// r30Server: a server with an ONLINE-by-report host "h1" of customer "c1", and an intent hub whose
|
||||
// presence for c1 is set up as asked. Real wall clock for "now" (the handler reads time.Now()).
|
||||
func r30Server(t *testing.T, p r30Presence) (*Server, *store.Store, *strings.Builder) {
|
||||
t.Helper()
|
||||
s, st := newTestServer(t)
|
||||
var logBuf strings.Builder
|
||||
s.logger = log.New(&logBuf, "", 0)
|
||||
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SaveHostReport("h1", "c1", []byte(`{}`), store.HostReportDenorm{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
hub := intent.New()
|
||||
var clock time.Time
|
||||
hub.SetClock(func() time.Time { return clock })
|
||||
set := func(d time.Duration) { clock = now.Add(d) }
|
||||
switch p {
|
||||
case r30Unknown:
|
||||
set(0)
|
||||
hub.SetClock(func() time.Time { return clock }) // born now
|
||||
case r30Connected:
|
||||
set(-time.Hour)
|
||||
hub.SetClock(func() time.Time { return clock })
|
||||
set(-time.Minute)
|
||||
hub.MarkWaitStart("c1")
|
||||
case r30OffLong:
|
||||
set(-time.Hour)
|
||||
hub.SetClock(func() time.Time { return clock })
|
||||
set(-15 * time.Minute)
|
||||
hub.MarkWaitStart("c1")
|
||||
set(-11 * time.Minute)
|
||||
hub.MarkWaitEnd("c1")
|
||||
case r30OffShort:
|
||||
set(-time.Hour)
|
||||
hub.SetClock(func() time.Time { return clock })
|
||||
set(-8 * time.Minute)
|
||||
hub.MarkWaitStart("c1")
|
||||
set(-5 * time.Minute)
|
||||
hub.MarkWaitEnd("c1")
|
||||
case r30NeverSeenLong:
|
||||
set(-20 * time.Minute)
|
||||
hub.SetClock(func() time.Time { return clock })
|
||||
}
|
||||
s.SetIntentHub(hub)
|
||||
if s.hostStatus(mustHost(t, st, "h1").LastReportAt) != "ok" {
|
||||
t.Fatal("fixture: host must read online by its report clock")
|
||||
}
|
||||
return s, st, &logBuf
|
||||
}
|
||||
|
||||
func mustHost(t *testing.T, st *store.Store, id string) *store.Host {
|
||||
t.Helper()
|
||||
h, err := st.GetHost(id)
|
||||
if err != nil || h == nil {
|
||||
t.Fatalf("GetHost %s: %v", id, err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func r30Form(tick bool) url.Values {
|
||||
f := url.Values{"confirm_host_id": {"h1"}, "delete_escrow": {"1"}}
|
||||
if tick {
|
||||
f.Set("box_off_confirmed", "1")
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// Online by report + no box connection for 11 min + the tick → deleted at once, logged with who, and
|
||||
// ONE operator event saved.
|
||||
func TestR30_OffLongWithTickDeletes(t *testing.T) {
|
||||
for _, p := range []r30Presence{r30OffLong, r30NeverSeenLong} {
|
||||
s, st, logBuf := r30Server(t, p)
|
||||
rr := postHostDelete(t, s, "h1", r30Form(true))
|
||||
if rr.Code != http.StatusSeeOther {
|
||||
t.Fatalf("presence %d: ticked delete = %d, want 303: %s", p, rr.Code, rr.Body.String())
|
||||
}
|
||||
if h, _ := st.GetHost("h1"); h != nil {
|
||||
t.Fatalf("presence %d: host still present after the box-off delete", p)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), `deleted while online by its report clock: operator`) ||
|
||||
!strings.Contains(logBuf.String(), `ticked "I checked: the box is off"`) {
|
||||
t.Errorf("presence %d: no audit line naming the tick:\n%s", p, logBuf.String())
|
||||
}
|
||||
evs, err := st.GetEventsByType("c1", hostDeletedBoxOffEvent, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(evs); n != 1 {
|
||||
t.Errorf("presence %d: %d %s events, want exactly 1", p, n, hostDeletedBoxOffEvent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every other combination is refused as today: 409, host still there, no event.
|
||||
func TestR30_RefusedCombinations(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
p r30Presence
|
||||
tick bool
|
||||
}{
|
||||
{"off long, no tick", r30OffLong, false},
|
||||
{"tick, box connected", r30Connected, true},
|
||||
{"tick, presence unknown (hub restarted)", r30Unknown, true},
|
||||
{"tick, not connected for less than the limit", r30OffShort, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s, st, _ := r30Server(t, c.p)
|
||||
rr := postHostDelete(t, s, "h1", r30Form(c.tick))
|
||||
if rr.Code != http.StatusConflict {
|
||||
t.Errorf("%s: = %d, want 409", c.name, rr.Code)
|
||||
}
|
||||
if h, _ := st.GetHost("h1"); h == nil {
|
||||
t.Errorf("%s: host DELETED despite the refusal", c.name)
|
||||
}
|
||||
if evs, _ := st.GetEventsByType("c1", hostDeletedBoxOffEvent, time.Now().Add(-time.Hour)); len(evs) != 0 {
|
||||
t.Errorf("%s: box-off event saved on a refusal", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No intent hub wired at all → the tick changes nothing (unknown presence).
|
||||
func TestR30_NoIntentHubTickRefused(t *testing.T) {
|
||||
s, st := newTestServer(t)
|
||||
_ = st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"})
|
||||
_ = st.SaveHostReport("h1", "c1", []byte(`{}`), store.HostReportDenorm{})
|
||||
if rr := postHostDelete(t, s, "h1", r30Form(true)); rr.Code != http.StatusConflict {
|
||||
t.Fatalf("no intent hub, ticked = %d, want 409", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// The fast-delete limit must stay at or above the presence window: a hub younger than the window
|
||||
// answers unknown, and a span shorter than one missed reconnect is not evidence of a dead box.
|
||||
func TestR30_OffDeleteLimitAboveWindow(t *testing.T) {
|
||||
if hostOffDeleteAfter < intent.PresenceConnectedWindow {
|
||||
t.Fatalf("hostOffDeleteAfter %s < presence window %s", hostOffDeleteAfter, intent.PresenceConnectedWindow)
|
||||
}
|
||||
}
|
||||
|
||||
func r30Render(t *testing.T, s *Server) string {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleHostDetail(rr, httptest.NewRequest(http.MethodGet, "/hosts/h1", nil), "h1")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("host detail = %d", rr.Code)
|
||||
}
|
||||
return rr.Body.String()
|
||||
}
|
||||
|
||||
// Render test per branch of both template gates (the seam-built-but-never-wired trap): the Box
|
||||
// connection line shows its state, and the danger zone + the tick box appear ONLY on the box-off branch.
|
||||
func TestR30_HostPageRendersEachBranch(t *testing.T) {
|
||||
cases := []struct {
|
||||
p r30Presence
|
||||
state string
|
||||
text string
|
||||
tickShow bool
|
||||
}{
|
||||
{r30Connected, "connected", "connected now", false},
|
||||
{r30Unknown, "unknown", "unknown (the hub restarted", false},
|
||||
{r30OffShort, "not_connected", "last connected", false},
|
||||
{r30OffLong, "not_connected", "last connected", true},
|
||||
{r30NeverSeenLong, "not_connected", "not connected since the hub started", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s, _, _ := r30Server(t, c.p)
|
||||
body := r30Render(t, s)
|
||||
if !strings.Contains(body, `data-box-connection="`+c.state+`"`) || !strings.Contains(body, c.text) {
|
||||
t.Errorf("presence %d: box connection line missing state %q / text %q", c.p, c.state, c.text)
|
||||
}
|
||||
hasTick := strings.Contains(body, `<input type="checkbox" id="host-delete-off-h1">`) && strings.Contains(body, "I checked: the box is off")
|
||||
hasZone := strings.Contains(body, "Danger zone")
|
||||
hasHidden := strings.Contains(body, `name="box_off_confirmed"`)
|
||||
if hasTick != c.tickShow || hasZone != c.tickShow || hasHidden != c.tickShow {
|
||||
t.Errorf("presence %d: tick=%t zone=%t hidden=%t, want all %t", c.p, hasTick, hasZone, hasHidden, c.tickShow)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A STALE host (deletable by the report clock) shows the danger zone WITHOUT the tick: the tick is
|
||||
// only for the online-by-report case.
|
||||
func TestR30_StaleHostNoTick(t *testing.T) {
|
||||
s, st := newTestServer(t)
|
||||
_ = st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"})
|
||||
body := r30Render(t, s) // never reported → "pending", deletable
|
||||
if !strings.Contains(body, "Danger zone") || strings.Contains(body, `id="host-delete-off-h1"`) || strings.Contains(body, `name="box_off_confirmed"`) {
|
||||
t.Fatalf("pending host: want danger zone without the tick")
|
||||
}
|
||||
}
|
||||
|
||||
// The confirm dialog's impact probe agrees with the page: deletable, with the tick required.
|
||||
func TestR30_ImpactJSONOffTick(t *testing.T) {
|
||||
s, _, _ := r30Server(t, r30OffLong)
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleHostDeleteImpact(rr, httptest.NewRequest(http.MethodGet, "/hosts/h1/delete-impact", nil), "h1")
|
||||
body := rr.Body.String()
|
||||
if !strings.Contains(body, `"deletable":true`) || !strings.Contains(body, `"off_tick_required":true`) {
|
||||
t.Fatalf("impact JSON = %s, want deletable + off_tick_required", body)
|
||||
}
|
||||
s2, _, _ := r30Server(t, r30Connected)
|
||||
rr = httptest.NewRecorder()
|
||||
s2.handleHostDeleteImpact(rr, httptest.NewRequest(http.MethodGet, "/hosts/h1/delete-impact", nil), "h1")
|
||||
if b := rr.Body.String(); !strings.Contains(b, `"deletable":false`) || !strings.Contains(b, `"off_tick_required":false`) {
|
||||
t.Fatalf("connected impact JSON = %s, want not deletable", b)
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,14 @@
|
||||
<span class="label">Last Report</span>
|
||||
<span class="value">{{if .HasReport}}{{timeAgoPtr .LastReportAt}}{{else}}waiting for first report{{end}}</span>
|
||||
</div>
|
||||
<!-- R-30 slice 1: the box's wait-channel presence (in memory; ~4-min cadence), beside the
|
||||
15-min report clock. Every branch pinned by r30_presence_delete_test.go. -->
|
||||
<div class="info-item">
|
||||
<span class="label">Box connection</span>
|
||||
{{if eq .BoxConnection.State "connected"}}<span class="value" data-box-connection="connected">connected now</span>
|
||||
{{else if eq .BoxConnection.State "not_connected"}}<span class="value" data-box-connection="not_connected" style="color: var(--warn)">{{if .BoxConnection.NeverSeen}}not connected since the hub started ({{.BoxConnection.Since.UTC.Format "2006-01-02 15:04"}} UTC, {{timeAgo .BoxConnection.Since}}){{else}}last connected {{.BoxConnection.Since.UTC.Format "2006-01-02 15:04"}} UTC ({{timeAgo .BoxConnection.Since}}){{end}}</span>
|
||||
{{else}}<span class="value" data-box-connection="unknown">unknown (the hub restarted recently, or the host has no customer)</span>{{end}}
|
||||
</div>
|
||||
<div class="info-item">
|
||||
<span class="label">Desired Generation</span>
|
||||
<span class="value">{{.DesiredGeneration}}</span>
|
||||
@@ -547,6 +555,8 @@
|
||||
{{if .Deletable}}
|
||||
<!-- Danger zone (v0.47.0): rendered ONLY for non-online hosts — deleting a live
|
||||
host would brick its heartbeat channel, so the affordance never exists for one.
|
||||
R-30 (D2): also for an online-by-report host with no box connection for 6+ minutes,
|
||||
behind the "I checked: the box is off" tick.
|
||||
Impact + type-to-confirm dialog per the global-floor confirm pattern. -->
|
||||
<section class="card" style="border-color: var(--crit);">
|
||||
<h2>Danger zone</h2>
|
||||
@@ -562,11 +572,21 @@
|
||||
<input type="checkbox" id="host-delete-escrow-{{.HostID}}">
|
||||
Move key escrow to retained custody (required when escrow present) + remove DR bundle for this host
|
||||
</label>
|
||||
{{if .OffTickRequired}}
|
||||
<!-- R-30 slice 2 (D2): online by its report clock, but no box connection for 6+ minutes.
|
||||
The server re-checks presence at POST time; the tick alone deletes nothing. -->
|
||||
<p style="margin: 0 0 0.5rem; font-size: 0.85em;">This host still reads online by its last report, but the hub has had no connection from the box for over 6 minutes.</p>
|
||||
<label id="host-delete-off-row-{{.HostID}}" style="display: block; margin: 0 0 0.5rem; font-size: 0.85em;">
|
||||
<input type="checkbox" id="host-delete-off-{{.HostID}}">
|
||||
I checked: the box is off
|
||||
</label>
|
||||
{{end}}
|
||||
<p style="margin: 0 0 0.5rem; font-size: 0.85em; color: var(--text-2);">Type the host id to confirm:</p>
|
||||
<form method="POST" action="/hosts/{{.HostID}}/delete" id="host-delete-form-{{.HostID}}" style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
|
||||
<input type="hidden" name="_csrf" value="{{.CSRFToken}}">
|
||||
<input type="hidden" name="confirm_host_id" id="host-delete-confirm-hidden-{{.HostID}}" value="">
|
||||
<input type="hidden" name="delete_escrow" id="host-delete-escrow-hidden-{{.HostID}}" value="">
|
||||
{{if .OffTickRequired}}<input type="hidden" name="box_off_confirmed" id="host-delete-off-hidden-{{.HostID}}" value="">{{end}}
|
||||
<input type="text" id="host-delete-confirm-input-{{.HostID}}" placeholder="retype the host id…" style="padding: 0.3em 0.5em; width: 16em;">
|
||||
<button type="button" class="btn btn-danger btn-sm" onclick="hostDeleteSubmit('{{.HostID}}')">Confirm & remove</button>
|
||||
<button type="button" class="btn btn-sm btn-outline" onclick="document.getElementById('host-delete-confirm-{{.HostID}}').style.display='none';">Cancel</button>
|
||||
@@ -591,7 +611,7 @@
|
||||
if (d.pbs_secret_present) parts.push('the staged PBS secret');
|
||||
if (d.recovery_present) parts.push('the break-glass recovery credential');
|
||||
impact.textContent = parts.join(', ') + '. Host status: ' + d.status + '.' +
|
||||
(d.deletable ? '' : ' Host is ONLINE — deletion will be refused.');
|
||||
(d.deletable ? (d.off_tick_required ? ' No box connection for 6+ minutes — the "box is off" tick is required.' : '') : ' Host is ONLINE — deletion will be refused.');
|
||||
document.getElementById('host-delete-escrow-row-' + hostID).style.display =
|
||||
d.escrow_present ? 'block' : 'none';
|
||||
})
|
||||
@@ -607,6 +627,12 @@
|
||||
err.textContent = 'This host has a key escrow — tick the acknowledgement to delete it too.';
|
||||
return;
|
||||
}
|
||||
var offCb = document.getElementById('host-delete-off-' + hostID);
|
||||
if (offCb && !offCb.checked) {
|
||||
err.textContent = 'Tick "I checked: the box is off" first.';
|
||||
return;
|
||||
}
|
||||
if (offCb) { document.getElementById('host-delete-off-hidden-' + hostID).value = '1'; }
|
||||
document.getElementById('host-delete-confirm-hidden-' + hostID).value = typed;
|
||||
document.getElementById('host-delete-escrow-hidden-' + hostID).value = escrowCb.checked ? '1' : '';
|
||||
document.getElementById('host-delete-form-' + hostID).submit();
|
||||
|
||||
Reference in New Issue
Block a user