380 lines
15 KiB
Go
380 lines
15 KiB
Go
package store
|
||
|
||
import (
|
||
"database/sql"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// Operator actions (R-314 / R-279 / R-177, `09` §3 decision 185 — D1, design option A of
|
||
// documentation/audits/day-2026-10-08/design-R-314-279-177.md).
|
||
//
|
||
// The operator presses a button on the host page; a row is stored here and the box's intent generation
|
||
// is bumped, so its wait channel wakes; the report ACK lists every pending row as
|
||
// `operator_actions: [{id, action, arg}]`; the controller acts once per id and sends
|
||
// `operator_action_results: [{id, outcome, message}]` on its next report; the row is closed and stops
|
||
// being listed. The hub never connects into the box.
|
||
//
|
||
// THE LIST IS CLOSED, and the hub refuses an unknown action, job or argument BEFORE storing anything —
|
||
// the controller refuses them again on its side. Nothing on the list deletes data, starts a countdown or
|
||
// shortens one; `03` §4 asks for a signing key only to destroy or overwrite the only copy. The list is
|
||
// pinned on both sides (TestOperatorActions_ClosedList here, TestOpActions_ClosedList in the controller)
|
||
// so a new entry is an operator decision, not an edit.
|
||
|
||
const (
|
||
OperatorActionOffsiteBackupNow = "offsite_backup_now"
|
||
OperatorActionAbandonStop = "abandon_stop"
|
||
OperatorActionAbandonExtend = "abandon_extend"
|
||
OperatorActionRunJob = "run_job"
|
||
// OperatorActionClearHold (D64 = B, operator 2026-10-29 12:10; `04` §6.1): the RUNNING controller clears one app's
|
||
// hold (any reason) and saves at once. No signature: the operator accepted that a compromised hub could release a
|
||
// data-protecting hold; the hub shows the hold's reason and a warning before the press.
|
||
OperatorActionClearHold = "clear_hold"
|
||
|
||
// HostOperatorActionRestartController (operator brief 2026-10-29, Part 1): the AGENT restarts a guest's controller
|
||
// (the supervisor's own act and guards). The agent's list holds this one verb.
|
||
HostOperatorActionRestartController = "restart_controller"
|
||
// HostOperatorActionRebootGuest (D65 = A, operator ruling 2026-10-29 15:50; `04` §6.1): the AGENT reboots one guest
|
||
// (`pct reboot` of THAT guest, never the host) with its own guards — no backup, restore, update or swap; at most
|
||
// once an hour; the restart brake. No signature: a reboot destroys nothing.
|
||
HostOperatorActionRebootGuest = "reboot_guest"
|
||
// HostOperatorActionRebootTTL: a reboot the agent did not pick up in 30 minutes expires — a box that was off must not
|
||
// reboot by itself when it comes back hours later.
|
||
HostOperatorActionRebootTTL = 30 * time.Minute
|
||
|
||
// Outcomes: the box's three, plus two the hub sets itself.
|
||
OperatorActionDone = "done"
|
||
OperatorActionRefused = "refused"
|
||
OperatorActionFailed = "failed"
|
||
OperatorActionExpired = "expired" // the box did not answer within OperatorActionTTL
|
||
OperatorActionCancelled = "cancelled" // the customer was RESET while it was pending
|
||
|
||
OperatorActionExtendMinDays = 1
|
||
OperatorActionExtendMaxDays = 30
|
||
|
||
// OperatorActionTTL: a pending action the box has not answered in a day is closed as expired and
|
||
// no longer listed — an off-site run pressed for a box that was off for a week must not start the
|
||
// moment it comes back, unasked.
|
||
OperatorActionTTL = 24 * time.Hour
|
||
|
||
// operatorActionMessageMax bounds the box's message stored per row.
|
||
operatorActionMessageMax = 500
|
||
// operatorActionsListed caps how many pending rows one ACK carries.
|
||
operatorActionsListed = 10
|
||
)
|
||
|
||
var operatorActionNames = []string{OperatorActionOffsiteBackupNow, OperatorActionAbandonStop, OperatorActionAbandonExtend, OperatorActionRunJob, OperatorActionClearHold}
|
||
|
||
// hostOperatorActionNames is the AGENT's closed list (the host report's reply). Pinned by
|
||
// TestHostOperatorActions_ClosedList; never shares a verb with the controller's list.
|
||
var hostOperatorActionNames = []string{HostOperatorActionRestartController, HostOperatorActionRebootGuest}
|
||
|
||
// stackNameRe is an app's stack name as the controller names it (a catalog slug).
|
||
var stackNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,63}$`)
|
||
|
||
// HostOperatorActionNames returns the agent's closed list (sorted copy).
|
||
func HostOperatorActionNames() []string { return sortedStrings(hostOperatorActionNames) }
|
||
|
||
// IsHostOperatorAction reports whether the verb belongs to the agent's list.
|
||
func IsHostOperatorAction(action string) bool { return inList(hostOperatorActionNames, action) }
|
||
|
||
// ValidateHostOperatorAction refuses anything outside the agent's closed list. Both verbs take the guest's vmid
|
||
// (Proxmox: 100–999999999).
|
||
func ValidateHostOperatorAction(action, arg string) error {
|
||
switch action {
|
||
case HostOperatorActionRestartController, HostOperatorActionRebootGuest:
|
||
v, err := strconv.Atoi(arg)
|
||
if err != nil || strconv.Itoa(v) != arg || v < 100 || v > 999999999 {
|
||
return fmt.Errorf("the guest id must be a number 100-999999999")
|
||
}
|
||
default:
|
||
return fmt.Errorf("unknown host action %q", action)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// operatorJobNames is the fixed set run_job may name — the controller's scheduler job names.
|
||
var operatorJobNames = []string{"fill-watch", "offsite-integrity", "offsite-proof", "disk-health-check"}
|
||
|
||
// OperatorActionNames returns the closed action list (sorted copy).
|
||
func OperatorActionNames() []string { return sortedStrings(operatorActionNames) }
|
||
|
||
// OperatorJobNames returns the fixed run_job set (sorted copy).
|
||
func OperatorJobNames() []string { return sortedStrings(operatorJobNames) }
|
||
|
||
func sortedStrings(in []string) []string {
|
||
out := append([]string(nil), in...)
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
func inList(list []string, v string) bool {
|
||
for _, x := range list {
|
||
if x == v {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ValidateOperatorAction refuses anything outside the closed list. A nil error is the only way a row
|
||
// is stored.
|
||
func ValidateOperatorAction(action, arg string) error {
|
||
switch action {
|
||
case OperatorActionOffsiteBackupNow, OperatorActionAbandonStop:
|
||
if arg != "" {
|
||
return fmt.Errorf("%s takes no argument", action)
|
||
}
|
||
case OperatorActionAbandonExtend:
|
||
d, err := strconv.Atoi(arg)
|
||
if err != nil || d < OperatorActionExtendMinDays || d > OperatorActionExtendMaxDays {
|
||
return fmt.Errorf("the number of days must be %d-%d", OperatorActionExtendMinDays, OperatorActionExtendMaxDays)
|
||
}
|
||
case OperatorActionRunJob:
|
||
if !inList(operatorJobNames, arg) {
|
||
return fmt.Errorf("unknown job %q", arg)
|
||
}
|
||
case OperatorActionClearHold:
|
||
if !stackNameRe.MatchString(arg) {
|
||
return fmt.Errorf("not an app name: %q", arg)
|
||
}
|
||
default:
|
||
return fmt.Errorf("unknown action %q", action)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// OperatorActionDirective is ONE entry of the report ACK's operator_actions list — the wire type,
|
||
// named so scripts/wire_contract_gate.py can check it field by field against the controller's
|
||
// report.OperatorAction.
|
||
type OperatorActionDirective struct {
|
||
ID int64 `json:"id"`
|
||
Action string `json:"action"`
|
||
Arg string `json:"arg,omitempty"`
|
||
}
|
||
|
||
// OperatorActionRow is one stored row, for the host page.
|
||
type OperatorActionRow struct {
|
||
ID int64
|
||
CustomerID string
|
||
HostID string // "" = a controller action; else the agent of this host acts
|
||
Action string
|
||
Arg string
|
||
RequestedAt time.Time
|
||
RequestedBy string
|
||
DoneAt *time.Time
|
||
Outcome string
|
||
Message string
|
||
}
|
||
|
||
func (s *Store) migrateOperatorActions() error {
|
||
_, err := s.db.Exec(`
|
||
CREATE TABLE IF NOT EXISTS operator_actions (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
customer_id TEXT NOT NULL,
|
||
action TEXT NOT NULL,
|
||
arg TEXT NOT NULL DEFAULT '',
|
||
requested_at DATETIME NOT NULL,
|
||
requested_by TEXT NOT NULL DEFAULT '',
|
||
done_at DATETIME,
|
||
outcome TEXT NOT NULL DEFAULT '',
|
||
message TEXT NOT NULL DEFAULT ''
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_operator_actions_customer ON operator_actions(customer_id, done_at);`)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// The agent's rows (2026-10-29): host_id names the host whose agent acts; "" = the controller's row.
|
||
if _, err := s.db.Exec(`ALTER TABLE operator_actions ADD COLUMN host_id TEXT NOT NULL DEFAULT ''`); err != nil &&
|
||
!strings.Contains(err.Error(), "duplicate column") {
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CreateHostOperatorAction validates against the AGENT's closed list and stores a pending row for hostID (listed on the
|
||
// customer's host page beside the controller's rows).
|
||
func (s *Store) CreateHostOperatorAction(customerID, hostID, action, arg, by string) (int64, error) {
|
||
if hostID == "" {
|
||
return 0, fmt.Errorf("host operator action: empty host id")
|
||
}
|
||
if err := ValidateHostOperatorAction(action, arg); err != nil {
|
||
return 0, err
|
||
}
|
||
res, err := s.db.Exec(`INSERT INTO operator_actions (customer_id, host_id, action, arg, requested_at, requested_by)
|
||
VALUES (?, ?, ?, ?, ?, ?)`, customerID, hostID, action, arg, time.Now().UTC(), by)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
return res.LastInsertId()
|
||
}
|
||
|
||
// PendingHostOperatorActions is what the next host-report reply lists for hostID's agent, oldest first; rows older
|
||
// than OperatorActionTTL are closed as expired first.
|
||
func (s *Store) PendingHostOperatorActions(hostID string) ([]OperatorActionDirective, error) {
|
||
now := time.Now().UTC()
|
||
if _, err := s.db.Exec(`UPDATE operator_actions SET done_at = ?, outcome = ?, message = ?
|
||
WHERE host_id = ? AND done_at IS NULL AND requested_at < ?`,
|
||
now, OperatorActionExpired, "the agent did not answer within a day", hostID, now.Add(-OperatorActionTTL)); err != nil {
|
||
return nil, err
|
||
}
|
||
if _, err := s.db.Exec(`UPDATE operator_actions SET done_at = ?, outcome = ?, message = ?
|
||
WHERE host_id = ? AND action = ? AND done_at IS NULL AND requested_at < ?`,
|
||
now, OperatorActionExpired, "the agent did not pick the reboot up within 30 minutes — press again if it is still needed",
|
||
hostID, HostOperatorActionRebootGuest, now.Add(-HostOperatorActionRebootTTL)); err != nil {
|
||
return nil, err
|
||
}
|
||
rows, err := s.db.Query(`SELECT id, action, arg FROM operator_actions
|
||
WHERE host_id = ? AND done_at IS NULL ORDER BY id LIMIT ?`, hostID, operatorActionsListed)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []OperatorActionDirective
|
||
for rows.Next() {
|
||
var d OperatorActionDirective
|
||
if err := rows.Scan(&d.ID, &d.Action, &d.Arg); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, d)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// RecordHostOperatorActionResult closes a pending AGENT row with the agent's result — matched on the id AND the
|
||
// reporting host, so another host's (or the controller's) result changes nothing.
|
||
func (s *Store) RecordHostOperatorActionResult(hostID string, id int64, outcome, message string) (*OperatorActionRow, error) {
|
||
switch outcome {
|
||
case OperatorActionDone, OperatorActionRefused, OperatorActionFailed:
|
||
default:
|
||
return nil, fmt.Errorf("unknown outcome %q", outcome)
|
||
}
|
||
if hostID == "" {
|
||
return nil, nil
|
||
}
|
||
if r := []rune(message); len(r) > operatorActionMessageMax {
|
||
message = string(r[:operatorActionMessageMax])
|
||
}
|
||
res, err := s.db.Exec(`UPDATE operator_actions SET done_at = ?, outcome = ?, message = ?
|
||
WHERE id = ? AND host_id = ? AND done_at IS NULL`, time.Now().UTC(), outcome, message, id, hostID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if n, _ := res.RowsAffected(); n == 0 {
|
||
return nil, nil
|
||
}
|
||
return s.getOperatorAction(id)
|
||
}
|
||
|
||
// CreateOperatorAction validates against the closed list and stores a pending row. by names who
|
||
// pressed (the operator's channel and address — never a credential).
|
||
func (s *Store) CreateOperatorAction(customerID, action, arg, by string) (int64, error) {
|
||
if customerID == "" {
|
||
return 0, fmt.Errorf("operator action: empty customer id")
|
||
}
|
||
if err := ValidateOperatorAction(action, arg); err != nil {
|
||
return 0, err
|
||
}
|
||
res, err := s.db.Exec(`INSERT INTO operator_actions (customer_id, action, arg, requested_at, requested_by)
|
||
VALUES (?, ?, ?, ?, ?)`, customerID, action, arg, time.Now().UTC(), by)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
return res.LastInsertId()
|
||
}
|
||
|
||
// PendingOperatorActions returns what the next ACK lists for this customer, oldest first. Rows older
|
||
// than OperatorActionTTL are closed as expired first and are not listed.
|
||
func (s *Store) PendingOperatorActions(customerID string) ([]OperatorActionDirective, error) {
|
||
now := time.Now().UTC()
|
||
if _, err := s.db.Exec(`UPDATE operator_actions SET done_at = ?, outcome = ?, message = ?
|
||
WHERE customer_id = ? AND host_id = '' AND done_at IS NULL AND requested_at < ?`,
|
||
now, OperatorActionExpired, "the box did not answer within a day", customerID, now.Add(-OperatorActionTTL)); err != nil {
|
||
return nil, err
|
||
}
|
||
rows, err := s.db.Query(`SELECT id, action, arg FROM operator_actions
|
||
WHERE customer_id = ? AND host_id = '' AND done_at IS NULL ORDER BY id LIMIT ?`, customerID, operatorActionsListed)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []OperatorActionDirective
|
||
for rows.Next() {
|
||
var d OperatorActionDirective
|
||
if err := rows.Scan(&d.ID, &d.Action, &d.Arg); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, d)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// RecordOperatorActionResult closes a pending row with the box's result. It matches on BOTH the id and
|
||
// the reporting customer, so a result naming another customer's action changes nothing (recorded=false).
|
||
// An outcome outside the box's three is refused. Returns the closed row when recorded.
|
||
func (s *Store) RecordOperatorActionResult(customerID string, id int64, outcome, message string) (*OperatorActionRow, error) {
|
||
switch outcome {
|
||
case OperatorActionDone, OperatorActionRefused, OperatorActionFailed:
|
||
default:
|
||
return nil, fmt.Errorf("unknown outcome %q", outcome)
|
||
}
|
||
if r := []rune(message); len(r) > operatorActionMessageMax {
|
||
message = string(r[:operatorActionMessageMax])
|
||
}
|
||
res, err := s.db.Exec(`UPDATE operator_actions SET done_at = ?, outcome = ?, message = ?
|
||
WHERE id = ? AND customer_id = ? AND host_id = '' AND done_at IS NULL`, time.Now().UTC(), outcome, message, id, customerID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if n, _ := res.RowsAffected(); n == 0 {
|
||
return nil, nil
|
||
}
|
||
return s.getOperatorAction(id)
|
||
}
|
||
|
||
func (s *Store) getOperatorAction(id int64) (*OperatorActionRow, error) {
|
||
rows, err := s.db.Query(`SELECT id, customer_id, host_id, action, arg, requested_at, requested_by, done_at, outcome, message
|
||
FROM operator_actions WHERE id = ?`, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
list, err := scanOperatorActions(rows)
|
||
if err != nil || len(list) == 0 {
|
||
return nil, err
|
||
}
|
||
return &list[0], nil
|
||
}
|
||
|
||
// ListOperatorActions returns the customer's newest rows (pending and closed), newest first.
|
||
func (s *Store) ListOperatorActions(customerID string, limit int) ([]OperatorActionRow, error) {
|
||
rows, err := s.db.Query(`SELECT id, customer_id, host_id, action, arg, requested_at, requested_by, done_at, outcome, message
|
||
FROM operator_actions WHERE customer_id = ? ORDER BY id DESC LIMIT ?`, customerID, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
return scanOperatorActions(rows)
|
||
}
|
||
|
||
func scanOperatorActions(rows *sql.Rows) ([]OperatorActionRow, error) {
|
||
var out []OperatorActionRow
|
||
for rows.Next() {
|
||
var r OperatorActionRow
|
||
var done sql.NullTime
|
||
if err := rows.Scan(&r.ID, &r.CustomerID, &r.HostID, &r.Action, &r.Arg, &r.RequestedAt, &r.RequestedBy, &done, &r.Outcome, &r.Message); err != nil {
|
||
return nil, err
|
||
}
|
||
if done.Valid {
|
||
t := done.Time
|
||
r.DoneAt = &t
|
||
}
|
||
out = append(out, r)
|
||
}
|
||
return out, rows.Err()
|
||
}
|