283 lines
13 KiB
Go
283 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// ── D64 = B + R-1343: the operator clears a hold in the RUNNING controller ─────────────────────────────────────────
|
|
//
|
|
// Two routes reach the same door (clearHoldNow):
|
|
// - the hub's „Clear hold" → the operator action clear_hold (internal/report/opactions.go), run in this process;
|
|
// - the box command `felhom-controller --clear-restore-hold <app>` (run with `docker exec` inside the guest) → the
|
|
// local socket below → the running controller.
|
|
// The running controller clears the hold in memory AND saves settings.json at once (backup.ClearHoldByOperator), so
|
|
// no later save — the hold reporter's, a hub report's — can write it back (R-1343). The household's „held" sentence
|
|
// goes with it: every reader (the page, the start gate, the API, the hold reporter) reads the same settings record.
|
|
// The ONE held flag kept elsewhere — stacks.Manager's „the last update ended held" — is forgotten here too.
|
|
|
|
// holdClearDeps are clearHoldNow's seams (production: holdClearDepsFor).
|
|
type holdClearDeps struct {
|
|
clear func(stack string, beforeLift func()) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperatorThen
|
|
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
|
|
// keepStopped (R-1350) records that the stop is wanted for an app that is not up — run once the hold is found and
|
|
// BEFORE it is lifted: stacks.Manager.RecordStopWantedIfDown (nil = none).
|
|
keepStopped func(stack string) (bool, error)
|
|
logger *log.Logger
|
|
}
|
|
|
|
// clearHoldNow lifts stack's hold and returns the result sentence (English, operator-facing). who names the person
|
|
// and the route for the box's own log, e.g. „the operator via the hub, action #12". A refusal (bad name, no hold) is
|
|
// wrapped in report.ErrActionRefused, and nothing was changed.
|
|
func clearHoldNow(d holdClearDeps, stack, who string) (string, error) {
|
|
if d.clear == nil {
|
|
return "", fmt.Errorf("%w: clearing a hold is not available on this box", report.ErrActionRefused)
|
|
}
|
|
var before func()
|
|
if d.keepStopped != nil {
|
|
before = func() {
|
|
rec, kerr := d.keepStopped(stack)
|
|
switch {
|
|
case kerr != nil && d.logger != nil:
|
|
d.logger.Printf("[WARN] [hold] %s: the stop could not be recorded as wanted (%v) — the dead-app alarm may follow the clear (R-1350)", stack, kerr)
|
|
case rec && d.logger != nil:
|
|
d.logger.Printf("[INFO] [hold] %s: the stop is recorded as wanted (desired state: stopped) — no dead-app alarm; the household's Start ends it (R-1350)", stack)
|
|
}
|
|
}
|
|
}
|
|
h, err := d.clear(stack, before)
|
|
switch {
|
|
case errors.Is(err, backup.ErrHoldBadStack):
|
|
return "", fmt.Errorf("%w: %q is not a valid app name — nothing was changed", report.ErrActionRefused, stack)
|
|
case errors.Is(err, backup.ErrNotHeld):
|
|
return "", fmt.Errorf("%w: %s has no hold — nothing was changed", report.ErrActionRefused, stack)
|
|
case err != nil:
|
|
return "", fmt.Errorf("clearing the hold on %s failed: %v", stack, err)
|
|
}
|
|
if d.forgetUpdateHeld != nil && d.forgetUpdateHeld(stack) {
|
|
if d.logger != nil {
|
|
d.logger.Printf("[INFO] [hold] %s: the update's held state is forgotten with the hold", stack)
|
|
}
|
|
}
|
|
if d.logger != nil {
|
|
d.logger.Printf("[WARN] [hold] %s: hold CLEARED by %s — it was %s (set %s); the app is not started by this", stack, who, h.WireReason(), h.At)
|
|
}
|
|
msg := fmt.Sprintf("the hold on %s is cleared (it was: %s). The app is NOT started by this — the household's Start, or a restore, starts it", stack, h.Why())
|
|
if h.Reason == settings.HoldReasonPutBackPending {
|
|
dir := h.SafetyDump
|
|
if dir == "" {
|
|
dir = "(not named in the hold)"
|
|
}
|
|
msg += ". Its waiting put-back is dropped: the pre-restore copy stays on disk at " + dir + " and is never poured back by itself"
|
|
}
|
|
return msg, nil
|
|
}
|
|
|
|
// ── the local channel (R-1343) ────────────────────────────────────────────────────────────────────────────────────
|
|
//
|
|
// WHY A UNIX SOCKET IN THE CONTROLLER'S DATA DIR, mode 0600 in a 0700 directory, and nothing on TCP:
|
|
// - traefik, the LAN and every app reach the controller only over TCP (:8080, :8081). A unix socket has no port,
|
|
// so none of them can reach it, whatever a proxy header or a forwarded request says;
|
|
// - the data dir is the controller's own volume: no app container mounts it, and only root in the guest
|
|
// (`pct exec <vmid> -- docker exec felhom-controller …`, the operator gate this project already uses) can enter
|
|
// the controller's container to use it;
|
|
// - the directory is created 0700 BEFORE the socket is bound, so the moment between bind and chmod is not a window;
|
|
// - the peer's uid must equal this process's (SO_PEERCRED, holdctl_linux.go) — a second lock on the same door.
|
|
// The customer's HTTP surface is deliberately NOT used: it authenticates as the household, and a household clearing
|
|
// its own hold is what the hold exists to prevent. Pinned by TestHoldCtl_NotReachableFromOutside.
|
|
|
|
const (
|
|
holdCtlDir = "ctl"
|
|
holdCtlSock = "hold.sock"
|
|
// holdCtlOpClear is the one request this channel answers.
|
|
holdCtlOpClear = "clear_hold"
|
|
// holdCtlMaxRequest bounds what one request may send.
|
|
holdCtlMaxRequest = 4096
|
|
)
|
|
|
|
// holdCtlPath is the socket's path under the controller's data dir.
|
|
func holdCtlPath(dataDir string) string { return filepath.Join(dataDir, holdCtlDir, holdCtlSock) }
|
|
|
|
type holdCtlRequest struct {
|
|
Op string `json:"op"`
|
|
Stack string `json:"stack"`
|
|
}
|
|
|
|
type holdCtlResponse struct {
|
|
Outcome string `json:"outcome"` // done | refused | failed (report.Outcome*)
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
// listenHoldCtl binds the socket and serves it until the listener is closed (closing it removes the socket file).
|
|
// clear is clearHoldNow bound to the running controller's deps.
|
|
func listenHoldCtl(path string, clear func(stack, who string) (string, error), logger *log.Logger) (net.Listener, error) {
|
|
dir := filepath.Dir(path)
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.Chmod(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
// A socket left by a controller that died is replaced; anything else at that path is refused, never removed.
|
|
if fi, err := os.Lstat(path); err == nil {
|
|
if fi.Mode()&os.ModeSocket == 0 {
|
|
return nil, fmt.Errorf("%s exists and is not a socket — refusing to replace it", path)
|
|
}
|
|
if err := os.Remove(path); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
ln, err := net.Listen("unix", path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.Chmod(path, 0o600); err != nil {
|
|
ln.Close()
|
|
return nil, err
|
|
}
|
|
go func() {
|
|
for {
|
|
conn, err := ln.Accept()
|
|
if err != nil {
|
|
if !errors.Is(err, net.ErrClosed) && logger != nil {
|
|
logger.Printf("[WARN] [hold] local channel stopped: %v", err)
|
|
}
|
|
return
|
|
}
|
|
go serveHoldCtl(conn, clear, logger)
|
|
}
|
|
}()
|
|
return ln, nil
|
|
}
|
|
|
|
func serveHoldCtl(conn net.Conn, clear func(stack, who string) (string, error), logger *log.Logger) {
|
|
defer conn.Close()
|
|
_ = conn.SetDeadline(time.Now().Add(30 * time.Second))
|
|
answer := func(r holdCtlResponse) { _ = json.NewEncoder(conn).Encode(r) }
|
|
if ok, why := holdCtlPeerAllowed(conn); !ok {
|
|
if logger != nil {
|
|
logger.Printf("[WARN] [hold] local channel: a request was refused — %s", why)
|
|
}
|
|
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "refused: " + why})
|
|
return
|
|
}
|
|
line, err := bufio.NewReader(io.LimitReader(conn, holdCtlMaxRequest)).ReadBytes('\n')
|
|
if err != nil && len(line) == 0 {
|
|
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "no request was read"})
|
|
return
|
|
}
|
|
var req holdCtlRequest
|
|
if err := json.Unmarshal(line, &req); err != nil || req.Op != holdCtlOpClear {
|
|
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "unknown request — this channel only clears a hold"})
|
|
return
|
|
}
|
|
msg, err := clear(req.Stack, "the operator on the box (--clear-restore-hold)")
|
|
switch {
|
|
case err == nil:
|
|
answer(holdCtlResponse{Outcome: report.OutcomeDone, Message: msg})
|
|
case errors.Is(err, report.ErrActionRefused):
|
|
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: strings.TrimPrefix(err.Error(), report.ErrActionRefused.Error()+": ")})
|
|
default:
|
|
answer(holdCtlResponse{Outcome: report.OutcomeFailed, Message: err.Error()})
|
|
}
|
|
}
|
|
|
|
// askRunningControllerToClear sends one clear request. reached=false: no controller answered at path (no socket, or
|
|
// a socket nobody listens on) — nothing was asked, so the caller may edit settings.json itself. reached=true with an
|
|
// error: the request may have been applied; the caller must not edit the file.
|
|
func askRunningControllerToClear(path, stack string) (resp holdCtlResponse, reached bool, err error) {
|
|
conn, err := net.DialTimeout("unix", path, 3*time.Second)
|
|
if err != nil {
|
|
return holdCtlResponse{}, false, err
|
|
}
|
|
defer conn.Close()
|
|
_ = conn.SetDeadline(time.Now().Add(60 * time.Second))
|
|
if err := json.NewEncoder(conn).Encode(holdCtlRequest{Op: holdCtlOpClear, Stack: stack}); err != nil {
|
|
return holdCtlResponse{}, true, err
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(conn, 64*1024)).Decode(&resp); err != nil {
|
|
return holdCtlResponse{}, true, err
|
|
}
|
|
return resp, true, nil
|
|
}
|
|
|
|
// clearRestoreHoldCLI is the box command `--clear-restore-hold <app>` (R-379, R-1343). It asks the running
|
|
// controller first; only when none answers does it edit settings.json itself, and it says which one happened.
|
|
// Returns the process exit code: 0 cleared, 2 refused (nothing changed), 1 failed or unknown.
|
|
func clearRestoreHoldCLI(dataDir, stack string, stdout, stderr io.Writer) int {
|
|
path := holdCtlPath(dataDir)
|
|
resp, reached, err := askRunningControllerToClear(path, stack)
|
|
if reached {
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "the running controller was reached, but its answer was lost: %v\n", err)
|
|
fmt.Fprintf(stderr, " The hold may or may not be cleared. Check with: felhom-controller --restore-holds\n")
|
|
fmt.Fprintf(stderr, " settings.json was NOT edited by this command.\n")
|
|
return 1
|
|
}
|
|
switch resp.Outcome {
|
|
case report.OutcomeDone:
|
|
fmt.Fprintf(stdout, "%s.\n", resp.Message)
|
|
fmt.Fprintf(stdout, " Cleared by the RUNNING controller (memory and settings.json together) — no restart is needed.\n")
|
|
return 0
|
|
case report.OutcomeRefused:
|
|
fmt.Fprintf(stderr, "%s\n", resp.Message)
|
|
return 2
|
|
default:
|
|
fmt.Fprintf(stderr, "the running controller could not clear the hold: %s\n", resp.Message)
|
|
return 1
|
|
}
|
|
}
|
|
// No controller answered: it is not running (or is older than this channel). The file is the only copy.
|
|
fmt.Fprintf(stdout, "No running controller answered on %s (%v).\n", path, err)
|
|
fmt.Fprintf(stdout, " Editing settings.json directly instead.\n")
|
|
if !backup.IsValidHoldStack(stack) {
|
|
fmt.Fprintf(stderr, "%q is not a valid app name — nothing was changed\n", stack)
|
|
return 2
|
|
}
|
|
lg := log.New(stderr, "", 0)
|
|
sett, lerr := settings.Load(filepath.Join(dataDir, "settings.json"), lg)
|
|
if lerr != nil {
|
|
fmt.Fprintf(stderr, "restore-hold: loading settings: %v\n", lerr)
|
|
return 1
|
|
}
|
|
// The same door as the running controller's (a waiting put-back is dropped from the app-stop marker too), over
|
|
// this process's copy — the only copy while no controller runs.
|
|
g := backup.NewAppStopGuard(filepath.Join(dataDir, "appstop-state.json"), lg)
|
|
g.SetHolds(sett)
|
|
h, cerr := g.ClearHoldByOperator(stack)
|
|
switch {
|
|
case errors.Is(cerr, backup.ErrNotHeld):
|
|
// "there was nothing to clear" is not success — a silent 0 here would let an operator believe they had
|
|
// unblocked an app whose name they mistyped.
|
|
fmt.Fprintf(stderr, "no restore hold in force for %q — nothing was changed\n", stack)
|
|
return 2
|
|
case cerr != nil:
|
|
fmt.Fprintf(stderr, "restore-hold: clearing %s: %v\n", stack, cerr)
|
|
return 1
|
|
}
|
|
fmt.Fprintf(stdout, "restore hold cleared for %s in settings.json (no controller was running to ask; it was: %s).\n", stack, h.Why())
|
|
if h.Reason == settings.HoldReasonPutBackPending {
|
|
fmt.Fprintf(stdout, " Its waiting put-back is dropped; the pre-restore copy stays on disk at %s.\n", orNone(h.SafetyDump))
|
|
}
|
|
fmt.Fprintf(stdout, " The controller reads it when it starts. If a controller IS running in this guest after all, restart it NOW,\n")
|
|
fmt.Fprintf(stdout, " INSIDE THE GUEST — from the Proxmox host: pct exec <vmid> -- systemctl restart felhom-controller-bootstrap.service\n")
|
|
fmt.Fprintf(stdout, " — or its next settings save writes the hold back (R-1343).\n")
|
|
return 0
|
|
}
|
|
|
|
// holdClearCommandHint is the exact guest command, for the help text and the operator's mail.
|
|
const holdClearCommandHint = "pct exec <vmid> -- docker exec felhom-controller felhom-controller --clear-restore-hold <app>"
|