package localapi import ( "context" "encoding/json" "errors" "fmt" "net/http" "os" "path/filepath" "strconv" "strings" "time" ) // reboot_guest — the operator reboots a guest from the hub (D65 = A, operator ruling 2026-10-29 15:50; felhom.eu `04` // §6.1, `03`). No signature: a reboot destroys nothing (`03`, „healing … non-destructive by construction"). // // WHY: a stuck Docker or a stuck network inside the box is out of reach of „Restart controller" — that restarts one // container. The reboot is the existing guest-reboot path (`pct reboot `, the household's „A Felhom // újraindítása", the same exact sudo vector) — of THAT guest, never the host. // // THE GUARDS (each refuses with the reason in the result; nothing is called): // - the guest is not one this agent provisioned, is not running, is locked, or its controller is parked; // - a controller swap (update) is in flight; // - a heavy operation holds the host-wide gate (a backup of any tier the agent runs, a restore test, an OS step), // or jobBlocker finds one (an agent backup job, a drive or NAS job, an escrow ceremony, a vzdump of the guest); // - the CONTROLLER says it is busy: a backup, a restore, an off-site run or an app update (its `ctl/busy` file, // read over one exact `pct exec … cat` grant; controllerBusy below). Fails closed on any read error but „no file"; // - at most ONE reboot per hour per guest (the record below, persisted — it survives an agent restart); // - the operator restart's brake: a reboot counts as a restart, and the brake's refusal refuses a reboot too. // The heavy-operation gate is HELD from the checks until the guest is up again, so no backup, restore test or OS // step of this agent can start in between. The guest-power watchdog and the controller supervisor leave the guest // alone while the reboot runs (rebootInFlight). // // AFTER: the result says when the guest was up and when the controller answered (both from the press). The record // `//operator-reboot.json` is written BEFORE the reboot, so the controller that starts after it finds // it: GET /guest/operator-reboot hands it to the household's dashboard („Felhom support restarted your box at …", // one day). Pinned by opreboot_test.go. const ( // OpRebootGuest is the agent's second operator verb (D65 = A). OpRebootGuest = "reboot_guest" // opRebootEvery: at most one operator reboot per guest per hour. opRebootEvery = time.Hour // opRebootNoticeFor: how long the household's dashboard shows the line. opRebootNoticeFor = 24 * time.Hour // opRebootTimeout bounds `pct reboot` (it blocks until the guest is back). opRebootTimeout = 5 * time.Minute operatorRebootFile = "operator-reboot.json" // ControllerBusyMarkerGuestPath is the controller's busy file as the GUEST sees it: the controller data volume // (`felhom-controller-data`, mounted at /opt/docker/felhom-controller in the container) + data/ctl/busy. One exact // sudo vector reads it (configs/felhom-agent.sudoers, FELHOM_CONTROLLERSWAP). It reads without Docker, so a stuck // Docker does not hide it. ControllerBusyMarkerGuestPath = "/var/lib/docker/volumes/felhom-controller-data/_data/data/ctl/busy" // controllerBusyFresh: the controller rewrites the file every 15 s while busy; an older file is a controller that // stopped writing (stuck or killed) — not a busy one. controllerBusyFresh = 90 * time.Second ) // operatorRebootRecord is the persisted record of the last operator reboot of a guest. type operatorRebootRecord struct { At time.Time `json:"at"` ActionID int64 `json:"action_id"` GuestUpAfterS int `json:"guest_up_after_s,omitempty"` ControllerAfterS int `json:"controller_after_s,omitempty"` } func (s *Server) operatorRebootPath(vmid int) string { return filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), operatorRebootFile) } // lastOperatorReboot reads the record; ok=false when there is none. A corrupt record is an error (the hourly limit // must not silently vanish). func (s *Server) lastOperatorReboot(vmid int) (operatorRebootRecord, bool, error) { raw, err := os.ReadFile(s.operatorRebootPath(vmid)) if errors.Is(err, os.ErrNotExist) { return operatorRebootRecord{}, false, nil } if err != nil { return operatorRebootRecord{}, false, err } var rec operatorRebootRecord if err := json.Unmarshal(raw, &rec); err != nil { return operatorRebootRecord{}, false, fmt.Errorf("corrupt %s: %w", operatorRebootFile, err) } return rec, !rec.At.IsZero(), nil } func (s *Server) saveOperatorReboot(vmid int, rec operatorRebootRecord) error { raw, err := json.Marshal(rec) if err != nil { return err } p := s.operatorRebootPath(vmid) tmp := p + ".tmp" if err := os.WriteFile(tmp, raw, 0o644); err != nil { return err } return os.Rename(tmp, p) } // rebooting reports whether an operator reboot of the guest is in flight (the watchdogs leave it alone). func (s *Server) rebooting(vmid int) bool { s.rebootMu.Lock() defer s.rebootMu.Unlock() return s.rebootInFlight[vmid] } func (s *Server) setRebooting(vmid int, on bool) { s.rebootMu.Lock() defer s.rebootMu.Unlock() if s.rebootInFlight == nil { s.rebootInFlight = map[int]bool{} } if on { s.rebootInFlight[vmid] = true } else { delete(s.rebootInFlight, vmid) } } func (s *Server) rebooter() interface { RebootGuest(ctx context.Context, vmid int) error } { if s.opRebooter != nil { return s.opRebooter } if s.guestAttach != nil { return s.guestAttach } return nil } // controllerBusy reads the controller's busy file. busy=true with the controller's reason when the file is fresh; // a missing file is „not busy" (an idle controller, or one older than the file); any other read failure is an error // (the caller refuses — it cannot prove no backup runs). func (s *Server) controllerBusy(ctx context.Context, vmid int) (bool, string, error) { out, err := s.guestExec.GuestExec(ctx, vmid, "cat", ControllerBusyMarkerGuestPath) if err != nil { if strings.Contains(strings.ToLower(err.Error()+" "+out), "no such file") { return false, "", nil } return false, "", err } lines := strings.SplitN(strings.TrimSpace(out), "\n", 2) at, perr := time.Parse(time.RFC3339, strings.TrimSpace(lines[0])) if perr != nil { return false, "", fmt.Errorf("the controller's busy file does not start with a time: %q", lines[0]) } if s.clock().Sub(at) > controllerBusyFresh { s.logger.Info("operator reboot: the controller's busy file is stale — the controller stopped writing it; not busy", "vmid", vmid, "written", at.UTC().Format(time.RFC3339)) return false, "", nil } why := "work in progress" if len(lines) == 2 && strings.TrimSpace(lines[1]) != "" { why = strings.TrimSpace(lines[1]) } return true, why, nil } // operatorRebootGuest is the press: every guard, then the reboot, then the wait for the guest and the controller. func (s *Server) operatorRebootGuest(ctx context.Context, id int64, arg string) (string, string) { vmid, err := strconv.Atoi(arg) if err != nil || strconv.Itoa(vmid) != arg { return opOutcomeRefused, fmt.Sprintf("refused: %q is not a guest id", arg) } rb := s.rebooter() if s.staleLock == nil || s.guestExec == nil || rb == nil { return opOutcomeRefused, "refused: the guest reboot is not wired on this agent" } if !s.provisionedGuest(vmid) { return opOutcomeRefused, fmt.Sprintf("refused: guest %d is not a guest this agent provisioned", vmid) } if s.rebooting(vmid) { return opOutcomeRefused, fmt.Sprintf("refused: a reboot of guest %d is already running", vmid) } guests, err := s.staleLock.Guests(ctx) if err != nil { return opOutcomeRefused, "refused: the guest list cannot be read — ownership unproven, nothing touched" } status := "" for _, g := range guests { if g.VMID == vmid { status = g.Status } } if status != "running" { return opOutcomeRefused, fmt.Sprintf("refused: guest %d is not running (%s) — the guest-power watchdog owns it", vmid, orDash(status)) } if s.controllerParked(vmid) { return opOutcomeRefused, fmt.Sprintf("refused: guest %d's controller is parked (remove the controller-parked marker first)", vmid) } // A controller swap (update) is refused by jobBlocker below, with the agent's other jobs. if lock, _, err := s.staleLock.Lock(ctx, vmid); err != nil { return opOutcomeRefused, "refused: the guest lock cannot be read — nothing touched" } else if lock != "" { return opOutcomeRefused, fmt.Sprintf("refused: guest %d is locked (%s) — another operation owns it", vmid, lock) } // At most one per hour. now := s.clock() last, had, err := s.lastOperatorReboot(vmid) if err != nil { return opOutcomeRefused, "refused: the last reboot's record cannot be read — " + err.Error() } if had && now.Sub(last.At) < opRebootEvery { return opOutcomeRefused, fmt.Sprintf("refused: guest %d was rebooted from the hub at %s UTC — at most one reboot per hour (next from %s UTC)", vmid, last.At.UTC().Format("15:04"), last.At.Add(opRebootEvery).UTC().Format("15:04")) } // The brake: a refusal in force refuses a reboot too, and 3 restarts in 15 minutes refuse it. s.ctrlSup.mu.Lock() st := s.supState(vmid) if now.Before(st.opBrakeUntil) { until := st.opBrakeUntil s.ctrlSup.mu.Unlock() return opOutcomeRefused, fmt.Sprintf("refused: 3 restarts in 15 minutes — presses are refused until %s UTC", until.UTC().Format("15:04")) } st.restarts = pruneBefore(st.restarts, now.Add(-controllerCrashloopWindow)) if len(st.restarts) >= controllerCrashloopMax { st.opBrakeUntil = now.Add(opBrakeRefusal) until := st.opBrakeUntil s.ctrlSup.mu.Unlock() return opOutcomeRefused, fmt.Sprintf("refused: 3 restarts in 15 minutes — presses are refused until %s UTC", until.UTC().Format("15:04")) } s.ctrlSup.mu.Unlock() // The heavy-operation gate, held until the guest is up again. release, busy, ok := s.inFlight.TryAcquire("guest-reboot") if !ok { return opOutcomeRefused, "refused: a backup or another heavy operation runs on the host (" + busy + ")" } defer release() if code, detail := s.jobBlocker(ctx, vmid); code != "" { return opOutcomeRefused, "refused: " + rebootBlockerSentence(code) + " (" + detail + ")" } if cbusy, why, err := s.controllerBusy(ctx, vmid); err != nil { return opOutcomeRefused, "refused: the controller's busy file cannot be read, so a running backup cannot be ruled out — " + err.Error() } else if cbusy { return opOutcomeRefused, "refused: the controller is busy — " + why } // The record FIRST: the hourly limit and the household's line both read it. rec := operatorRebootRecord{At: now, ActionID: id} if err := s.saveOperatorReboot(vmid, rec); err != nil { return opOutcomeRefused, "refused: the reboot cannot be recorded — " + err.Error() } s.ctrlSup.mu.Lock() st = s.supState(vmid) st.restarts = append(st.restarts, now) st.restartsTotal++ st.lastRestartAt = now st.lastReason = fmt.Sprintf("guest reboot by the operator via the hub (action #%d)", id) st.notRunningSeen = 0 s.ctrlSup.mu.Unlock() s.setRebooting(vmid, true) defer s.setRebooting(vmid, false) s.logger.Warn("operator reboot: REBOOTING the guest on the operator's press", "vmid", vmid, "action_id", id) began := time.Now() rctx, cancel := context.WithTimeout(ctx, opRebootTimeout) err = rb.RebootGuest(rctx, vmid) cancel() if err != nil { return opOutcomeFailed, fmt.Sprintf("the reboot of guest %d failed: %v — look at the box (the guest-power watchdog starts a stopped guest)", vmid, err) } upS := int(time.Since(began).Round(time.Second) / time.Second) rec.GuestUpAfterS = upS // The controller's own HEALTHCHECK is the answer, as for a restart; the wait starts when the guest is up. for deadline := time.Now().Add(s.opWait()); time.Now().Before(deadline); { out, err := s.guestExec.GuestExec(ctx, vmid, "docker", "inspect", "-f", opHealthTemplate, controllerContainer) if err == nil { f := strings.SplitN(strings.TrimSpace(out), "|", 2) if len(f) == 2 && f[0] == "true" && (f[1] == "healthy" || f[1] == "none") { rec.ControllerAfterS = int(time.Since(began).Round(time.Second) / time.Second) if serr := s.saveOperatorReboot(vmid, rec); serr != nil { s.logger.Warn("operator reboot: the times could not be added to the record", "vmid", vmid, "err", serr) } return opOutcomeDone, fmt.Sprintf("rebooted; the guest was up after %d s, the controller answered after %d s", upS, rec.ControllerAfterS) } } select { case <-ctx.Done(): return opOutcomeFailed, fmt.Sprintf("rebooted; the guest was up after %d s, but the wait for the controller was cut", upS) case <-time.After(s.opPoll()): } } _ = s.saveOperatorReboot(vmid, rec) return opOutcomeFailed, fmt.Sprintf("rebooted; the guest was up after %d s, but the controller gave no healthy answer within %s — look at the box", upS, s.opWait().Round(time.Second)) } // rebootBlockerSentence names jobBlocker's code in the result's words. func rebootBlockerSentence(code string) string { switch code { case PowerOffBackup: return "a backup is running" case PowerOffCtrlUpdate: return "a controller update is running" case PowerOffDrive: return "a drive or network-storage job is running" case PowerOffEscrow: return "an escrow ceremony is running" case PowerOffUnknown: return "the running jobs cannot be read" default: return "the host is busy" } } // handleOperatorRebootNotice — GET /guest/operator-reboot (self-scoped): the last operator reboot of THIS guest, when it // is less than a day old, for the household's dashboard line. {"at": ""} when there is none. func (s *Server) handleOperatorRebootNotice(w http.ResponseWriter, _ *http.Request, vmid int) { rec, ok, err := s.lastOperatorReboot(vmid) if err != nil { writeErr(w, http.StatusInternalServerError, "the reboot record cannot be read") return } if !ok || s.clock().Sub(rec.At) >= opRebootNoticeFor { writeOK(w, map[string]any{"vmid": vmid, "at": ""}) return } writeOK(w, map[string]any{"vmid": vmid, "at": rec.At.UTC().Format(time.RFC3339), "by": "operator"}) }