felhom-agent/internal/localapi/opreboot.go

package localapi

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// reboot_guest — the operator reboots a guest from the hub (D65 = A, operator ruling 2026-10-29 15:50; felhom.eu `04`
// §6.1, `03`). No signature: a reboot destroys nothing (`03`, „healing … non-destructive by construction").
//
// WHY: a stuck Docker or a stuck network inside the box is out of reach of „Restart controller" — that restarts one
// container. The reboot is the existing guest-reboot path (`pct reboot <vmid>`, the household's „A Felhom
// újraindítása", the same exact sudo vector) — of THAT guest, never the host.
//
// THE GUARDS (each refuses with the reason in the result; nothing is called):
//   - the guest is not one this agent provisioned, is not running, is locked, or its controller is parked;
//   - a controller swap (update) is in flight;
//   - a heavy operation holds the host-wide gate (a backup of any tier the agent runs, a restore test, an OS step),
//     or jobBlocker finds one (an agent backup job, a drive or NAS job, an escrow ceremony, a vzdump of the guest);
//   - the CONTROLLER says it is busy: a backup, a restore, an off-site run or an app update (its `ctl/busy` file,
//     read over one exact `pct exec … cat` grant; controllerBusy below). Fails closed on any read error but „no file";
//   - at most ONE reboot per hour per guest (the record below, persisted — it survives an agent restart);
//   - the operator restart's brake: a reboot counts as a restart, and the brake's refusal refuses a reboot too.
// The heavy-operation gate is HELD from the checks until the guest is up again, so no backup, restore test or OS
// step of this agent can start in between. The guest-power watchdog and the controller supervisor leave the guest
// alone while the reboot runs (rebootInFlight).
//
// AFTER: the result says when the guest was up and when the controller answered (both from the press). The record
// `<guests>/<vmid>/operator-reboot.json` is written BEFORE the reboot, so the controller that starts after it finds
// it: GET /guest/operator-reboot hands it to the household's dashboard („Felhom support restarted your box at …",
// one day). Pinned by opreboot_test.go.

const (
	// OpRebootGuest is the agent's second operator verb (D65 = A).
	OpRebootGuest = "reboot_guest"

	// opRebootEvery: at most one operator reboot per guest per hour.
	opRebootEvery = time.Hour
	// opRebootNoticeFor: how long the household's dashboard shows the line.
	opRebootNoticeFor = 24 * time.Hour
	// opRebootTimeout bounds `pct reboot` (it blocks until the guest is back).
	opRebootTimeout = 5 * time.Minute

	operatorRebootFile = "operator-reboot.json"

	// ControllerBusyMarkerGuestPath is the controller's busy file as the GUEST sees it: the controller data volume
	// (`felhom-controller-data`, mounted at /opt/docker/felhom-controller in the container) + data/ctl/busy. One exact
	// sudo vector reads it (configs/felhom-agent.sudoers, FELHOM_CONTROLLERSWAP). It reads without Docker, so a stuck
	// Docker does not hide it.
	ControllerBusyMarkerGuestPath = "/var/lib/docker/volumes/felhom-controller-data/_data/data/ctl/busy"
	// controllerBusyFresh: the controller rewrites the file every 15 s while busy; an older file is a controller that
	// stopped writing (stuck or killed) — not a busy one.
	controllerBusyFresh = 90 * time.Second
)

// operatorRebootRecord is the persisted record of the last operator reboot of a guest.
type operatorRebootRecord struct {
	At               time.Time `json:"at"`
	ActionID         int64     `json:"action_id"`
	GuestUpAfterS    int       `json:"guest_up_after_s,omitempty"`
	ControllerAfterS int       `json:"controller_after_s,omitempty"`
}

func (s *Server) operatorRebootPath(vmid int) string {
	return filepath.Join(s.guestsStateDir(), strconv.Itoa(vmid), operatorRebootFile)
}

// lastOperatorReboot reads the record; ok=false when there is none. A corrupt record is an error (the hourly limit
// must not silently vanish).
func (s *Server) lastOperatorReboot(vmid int) (operatorRebootRecord, bool, error) {
	raw, err := os.ReadFile(s.operatorRebootPath(vmid))
	if errors.Is(err, os.ErrNotExist) {
		return operatorRebootRecord{}, false, nil
	}
	if err != nil {
		return operatorRebootRecord{}, false, err
	}
	var rec operatorRebootRecord
	if err := json.Unmarshal(raw, &rec); err != nil {
		return operatorRebootRecord{}, false, fmt.Errorf("corrupt %s: %w", operatorRebootFile, err)
	}
	return rec, !rec.At.IsZero(), nil
}

func (s *Server) saveOperatorReboot(vmid int, rec operatorRebootRecord) error {
	raw, err := json.Marshal(rec)
	if err != nil {
		return err
	}
	p := s.operatorRebootPath(vmid)
	tmp := p + ".tmp"
	if err := os.WriteFile(tmp, raw, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, p)
}

// rebooting reports whether an operator reboot of the guest is in flight (the watchdogs leave it alone).
func (s *Server) rebooting(vmid int) bool {
	s.rebootMu.Lock()
	defer s.rebootMu.Unlock()
	return s.rebootInFlight[vmid]
}

func (s *Server) setRebooting(vmid int, on bool) {
	s.rebootMu.Lock()
	defer s.rebootMu.Unlock()
	if s.rebootInFlight == nil {
		s.rebootInFlight = map[int]bool{}
	}
	if on {
		s.rebootInFlight[vmid] = true
	} else {
		delete(s.rebootInFlight, vmid)
	}
}

func (s *Server) rebooter() interface {
	RebootGuest(ctx context.Context, vmid int) error
} {
	if s.opRebooter != nil {
		return s.opRebooter
	}
	if s.guestAttach != nil {
		return s.guestAttach
	}
	return nil
}

// controllerBusy reads the controller's busy file. busy=true with the controller's reason when the file is fresh;
// a missing file is „not busy" (an idle controller, or one older than the file); any other read failure is an error
// (the caller refuses — it cannot prove no backup runs).
func (s *Server) controllerBusy(ctx context.Context, vmid int) (bool, string, error) {
	out, err := s.guestExec.GuestExec(ctx, vmid, "cat", ControllerBusyMarkerGuestPath)
	if err != nil {
		if strings.Contains(strings.ToLower(err.Error()+" "+out), "no such file") {
			return false, "", nil
		}
		return false, "", err
	}
	lines := strings.SplitN(strings.TrimSpace(out), "\n", 2)
	at, perr := time.Parse(time.RFC3339, strings.TrimSpace(lines[0]))
	if perr != nil {
		return false, "", fmt.Errorf("the controller's busy file does not start with a time: %q", lines[0])
	}
	if s.clock().Sub(at) > controllerBusyFresh {
		s.logger.Info("operator reboot: the controller's busy file is stale — the controller stopped writing it; not busy",
			"vmid", vmid, "written", at.UTC().Format(time.RFC3339))
		return false, "", nil
	}
	why := "work in progress"
	if len(lines) == 2 && strings.TrimSpace(lines[1]) != "" {
		why = strings.TrimSpace(lines[1])
	}
	return true, why, nil
}

// operatorRebootGuest is the press: every guard, then the reboot, then the wait for the guest and the controller.
func (s *Server) operatorRebootGuest(ctx context.Context, id int64, arg string) (string, string) {
	vmid, err := strconv.Atoi(arg)
	if err != nil || strconv.Itoa(vmid) != arg {
		return opOutcomeRefused, fmt.Sprintf("refused: %q is not a guest id", arg)
	}
	rb := s.rebooter()
	if s.staleLock == nil || s.guestExec == nil || rb == nil {
		return opOutcomeRefused, "refused: the guest reboot is not wired on this agent"
	}
	if !s.provisionedGuest(vmid) {
		return opOutcomeRefused, fmt.Sprintf("refused: guest %d is not a guest this agent provisioned", vmid)
	}
	if s.rebooting(vmid) {
		return opOutcomeRefused, fmt.Sprintf("refused: a reboot of guest %d is already running", vmid)
	}
	guests, err := s.staleLock.Guests(ctx)
	if err != nil {
		return opOutcomeRefused, "refused: the guest list cannot be read — ownership unproven, nothing touched"
	}
	status := ""
	for _, g := range guests {
		if g.VMID == vmid {
			status = g.Status
		}
	}
	if status != "running" {
		return opOutcomeRefused, fmt.Sprintf("refused: guest %d is not running (%s) — the guest-power watchdog owns it", vmid, orDash(status))
	}
	if s.controllerParked(vmid) {
		return opOutcomeRefused, fmt.Sprintf("refused: guest %d's controller is parked (remove the controller-parked marker first)", vmid)
	}
	// A controller swap (update) is refused by jobBlocker below, with the agent's other jobs.
	if lock, _, err := s.staleLock.Lock(ctx, vmid); err != nil {
		return opOutcomeRefused, "refused: the guest lock cannot be read — nothing touched"
	} else if lock != "" {
		return opOutcomeRefused, fmt.Sprintf("refused: guest %d is locked (%s) — another operation owns it", vmid, lock)
	}

	// At most one per hour.
	now := s.clock()
	last, had, err := s.lastOperatorReboot(vmid)
	if err != nil {
		return opOutcomeRefused, "refused: the last reboot's record cannot be read — " + err.Error()
	}
	if had && now.Sub(last.At) < opRebootEvery {
		return opOutcomeRefused, fmt.Sprintf("refused: guest %d was rebooted from the hub at %s UTC — at most one reboot per hour (next from %s UTC)",
			vmid, last.At.UTC().Format("15:04"), last.At.Add(opRebootEvery).UTC().Format("15:04"))
	}

	// The brake: a refusal in force refuses a reboot too, and 3 restarts in 15 minutes refuse it.
	s.ctrlSup.mu.Lock()
	st := s.supState(vmid)
	if now.Before(st.opBrakeUntil) {
		until := st.opBrakeUntil
		s.ctrlSup.mu.Unlock()
		return opOutcomeRefused, fmt.Sprintf("refused: 3 restarts in 15 minutes — presses are refused until %s UTC", until.UTC().Format("15:04"))
	}
	st.restarts = pruneBefore(st.restarts, now.Add(-controllerCrashloopWindow))
	if len(st.restarts) >= controllerCrashloopMax {
		st.opBrakeUntil = now.Add(opBrakeRefusal)
		until := st.opBrakeUntil
		s.ctrlSup.mu.Unlock()
		return opOutcomeRefused, fmt.Sprintf("refused: 3 restarts in 15 minutes — presses are refused until %s UTC", until.UTC().Format("15:04"))
	}
	s.ctrlSup.mu.Unlock()

	// The heavy-operation gate, held until the guest is up again.
	release, busy, ok := s.inFlight.TryAcquire("guest-reboot")
	if !ok {
		return opOutcomeRefused, "refused: a backup or another heavy operation runs on the host (" + busy + ")"
	}
	defer release()
	if code, detail := s.jobBlocker(ctx, vmid); code != "" {
		return opOutcomeRefused, "refused: " + rebootBlockerSentence(code) + " (" + detail + ")"
	}
	if cbusy, why, err := s.controllerBusy(ctx, vmid); err != nil {
		return opOutcomeRefused, "refused: the controller's busy file cannot be read, so a running backup cannot be ruled out — " + err.Error()
	} else if cbusy {
		return opOutcomeRefused, "refused: the controller is busy — " + why
	}

	// The record FIRST: the hourly limit and the household's line both read it.
	rec := operatorRebootRecord{At: now, ActionID: id}
	if err := s.saveOperatorReboot(vmid, rec); err != nil {
		return opOutcomeRefused, "refused: the reboot cannot be recorded — " + err.Error()
	}
	s.ctrlSup.mu.Lock()
	st = s.supState(vmid)
	st.restarts = append(st.restarts, now)
	st.restartsTotal++
	st.lastRestartAt = now
	st.lastReason = fmt.Sprintf("guest reboot by the operator via the hub (action #%d)", id)
	st.notRunningSeen = 0
	s.ctrlSup.mu.Unlock()
	s.setRebooting(vmid, true)
	defer s.setRebooting(vmid, false)

	s.logger.Warn("operator reboot: REBOOTING the guest on the operator's press", "vmid", vmid, "action_id", id)
	began := time.Now()
	rctx, cancel := context.WithTimeout(ctx, opRebootTimeout)
	err = rb.RebootGuest(rctx, vmid)
	cancel()
	if err != nil {
		return opOutcomeFailed, fmt.Sprintf("the reboot of guest %d failed: %v — look at the box (the guest-power watchdog starts a stopped guest)", vmid, err)
	}
	upS := int(time.Since(began).Round(time.Second) / time.Second)
	rec.GuestUpAfterS = upS
	// The controller's own HEALTHCHECK is the answer, as for a restart; the wait starts when the guest is up.
	for deadline := time.Now().Add(s.opWait()); time.Now().Before(deadline); {
		out, err := s.guestExec.GuestExec(ctx, vmid, "docker", "inspect", "-f", opHealthTemplate, controllerContainer)
		if err == nil {
			f := strings.SplitN(strings.TrimSpace(out), "|", 2)
			if len(f) == 2 && f[0] == "true" && (f[1] == "healthy" || f[1] == "none") {
				rec.ControllerAfterS = int(time.Since(began).Round(time.Second) / time.Second)
				if serr := s.saveOperatorReboot(vmid, rec); serr != nil {
					s.logger.Warn("operator reboot: the times could not be added to the record", "vmid", vmid, "err", serr)
				}
				return opOutcomeDone, fmt.Sprintf("rebooted; the guest was up after %d s, the controller answered after %d s", upS, rec.ControllerAfterS)
			}
		}
		select {
		case <-ctx.Done():
			return opOutcomeFailed, fmt.Sprintf("rebooted; the guest was up after %d s, but the wait for the controller was cut", upS)
		case <-time.After(s.opPoll()):
		}
	}
	_ = s.saveOperatorReboot(vmid, rec)
	return opOutcomeFailed, fmt.Sprintf("rebooted; the guest was up after %d s, but the controller gave no healthy answer within %s — look at the box", upS, s.opWait().Round(time.Second))
}

// rebootBlockerSentence names jobBlocker's code in the result's words.
func rebootBlockerSentence(code string) string {
	switch code {
	case PowerOffBackup:
		return "a backup is running"
	case PowerOffCtrlUpdate:
		return "a controller update is running"
	case PowerOffDrive:
		return "a drive or network-storage job is running"
	case PowerOffEscrow:
		return "an escrow ceremony is running"
	case PowerOffUnknown:
		return "the running jobs cannot be read"
	default:
		return "the host is busy"
	}
}

// handleOperatorRebootNotice — GET /guest/operator-reboot (self-scoped): the last operator reboot of THIS guest, when it
// is less than a day old, for the household's dashboard line. {"at": ""} when there is none.
func (s *Server) handleOperatorRebootNotice(w http.ResponseWriter, _ *http.Request, vmid int) {
	rec, ok, err := s.lastOperatorReboot(vmid)
	if err != nil {
		writeErr(w, http.StatusInternalServerError, "the reboot record cannot be read")
		return
	}
	if !ok || s.clock().Sub(rec.At) >= opRebootNoticeFor {
		writeOK(w, map[string]any{"vmid": vmid, "at": ""})
		return
	}
	writeOK(w, map[string]any{"vmid": vmid, "at": rec.At.UTC().Format(time.RFC3339), "by": "operator"})
}