clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type. - --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart). - R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause, failed put-back step, database replay or not, kept copy) — never the option-C database story by default. - R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
This commit is contained in:
1 parent
c7bb776e00
commit
5b3b541602
25 files changed
+1809
-83
No files matched your search
@@ -0,0 +1,260 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── D64 = B + R-1343: the operator clears a hold in the RUNNING controller ─────────────────────────────────────────
|
||||
//
|
||||
// Two routes reach the same door (clearHoldNow):
|
||||
// - the hub's „Clear hold" → the operator action clear_hold (internal/report/opactions.go), run in this process;
|
||||
// - the box command `felhom-controller --clear-restore-hold <app>` (run with `docker exec` inside the guest) → the
|
||||
// local socket below → the running controller.
|
||||
// The running controller clears the hold in memory AND saves settings.json at once (backup.ClearHoldByOperator), so
|
||||
// no later save — the hold reporter's, a hub report's — can write it back (R-1343). The household's „held" sentence
|
||||
// goes with it: every reader (the page, the start gate, the API, the hold reporter) reads the same settings record.
|
||||
// The ONE held flag kept elsewhere — stacks.Manager's „the last update ended held" — is forgotten here too.
|
||||
|
||||
// holdClearDeps are clearHoldNow's seams (production: holdClearDepsFor).
|
||||
type holdClearDeps struct {
|
||||
clear func(stack string) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperator
|
||||
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// clearHoldNow lifts stack's hold and returns the result sentence (English, operator-facing). who names the person
|
||||
// and the route for the box's own log, e.g. „the operator via the hub, action #12". A refusal (bad name, no hold) is
|
||||
// wrapped in report.ErrActionRefused, and nothing was changed.
|
||||
func clearHoldNow(d holdClearDeps, stack, who string) (string, error) {
|
||||
if d.clear == nil {
|
||||
return "", fmt.Errorf("%w: clearing a hold is not available on this box", report.ErrActionRefused)
|
||||
}
|
||||
h, err := d.clear(stack)
|
||||
switch {
|
||||
case errors.Is(err, backup.ErrHoldBadStack):
|
||||
return "", fmt.Errorf("%w: %q is not a valid app name — nothing was changed", report.ErrActionRefused, stack)
|
||||
case errors.Is(err, backup.ErrNotHeld):
|
||||
return "", fmt.Errorf("%w: %s has no hold — nothing was changed", report.ErrActionRefused, stack)
|
||||
case err != nil:
|
||||
return "", fmt.Errorf("clearing the hold on %s failed: %v", stack, err)
|
||||
}
|
||||
if d.forgetUpdateHeld != nil && d.forgetUpdateHeld(stack) {
|
||||
if d.logger != nil {
|
||||
d.logger.Printf("[INFO] [hold] %s: the update's held state is forgotten with the hold", stack)
|
||||
}
|
||||
}
|
||||
if d.logger != nil {
|
||||
d.logger.Printf("[WARN] [hold] %s: hold CLEARED by %s — it was %s (set %s); the app is not started by this", stack, who, h.WireReason(), h.At)
|
||||
}
|
||||
msg := fmt.Sprintf("the hold on %s is cleared (it was: %s). The app is NOT started by this — the household's Start, or a restore, starts it", stack, h.Why())
|
||||
if h.Reason == settings.HoldReasonPutBackPending {
|
||||
dir := h.SafetyDump
|
||||
if dir == "" {
|
||||
dir = "(not named in the hold)"
|
||||
}
|
||||
msg += ". Its waiting put-back is dropped: the pre-restore copy stays on disk at " + dir + " and is never poured back by itself"
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// ── the local channel (R-1343) ────────────────────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// WHY A UNIX SOCKET IN THE CONTROLLER'S DATA DIR, mode 0600 in a 0700 directory, and nothing on TCP:
|
||||
// - traefik, the LAN and every app reach the controller only over TCP (:8080, :8081). A unix socket has no port,
|
||||
// so none of them can reach it, whatever a proxy header or a forwarded request says;
|
||||
// - the data dir is the controller's own volume: no app container mounts it, and only root in the guest
|
||||
// (`pct exec <vmid> -- docker exec felhom-controller …`, the operator gate this project already uses) can enter
|
||||
// the controller's container to use it;
|
||||
// - the directory is created 0700 BEFORE the socket is bound, so the moment between bind and chmod is not a window;
|
||||
// - the peer's uid must equal this process's (SO_PEERCRED, holdctl_linux.go) — a second lock on the same door.
|
||||
// The customer's HTTP surface is deliberately NOT used: it authenticates as the household, and a household clearing
|
||||
// its own hold is what the hold exists to prevent. Pinned by TestHoldCtl_NotReachableFromOutside.
|
||||
|
||||
const (
|
||||
holdCtlDir = "ctl"
|
||||
holdCtlSock = "hold.sock"
|
||||
// holdCtlOpClear is the one request this channel answers.
|
||||
holdCtlOpClear = "clear_hold"
|
||||
// holdCtlMaxRequest bounds what one request may send.
|
||||
holdCtlMaxRequest = 4096
|
||||
)
|
||||
|
||||
// holdCtlPath is the socket's path under the controller's data dir.
|
||||
func holdCtlPath(dataDir string) string { return filepath.Join(dataDir, holdCtlDir, holdCtlSock) }
|
||||
|
||||
type holdCtlRequest struct {
|
||||
Op string `json:"op"`
|
||||
Stack string `json:"stack"`
|
||||
}
|
||||
|
||||
type holdCtlResponse struct {
|
||||
Outcome string `json:"outcome"` // done | refused | failed (report.Outcome*)
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// listenHoldCtl binds the socket and serves it until the listener is closed (closing it removes the socket file).
|
||||
// clear is clearHoldNow bound to the running controller's deps.
|
||||
func listenHoldCtl(path string, clear func(stack, who string) (string, error), logger *log.Logger) (net.Listener, error) {
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A socket left by a controller that died is replaced; anything else at that path is refused, never removed.
|
||||
if fi, err := os.Lstat(path); err == nil {
|
||||
if fi.Mode()&os.ModeSocket == 0 {
|
||||
return nil, fmt.Errorf("%s exists and is not a socket — refusing to replace it", path)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
ln, err := net.Listen("unix", path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
ln.Close()
|
||||
return nil, err
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
if !errors.Is(err, net.ErrClosed) && logger != nil {
|
||||
logger.Printf("[WARN] [hold] local channel stopped: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
go serveHoldCtl(conn, clear, logger)
|
||||
}
|
||||
}()
|
||||
return ln, nil
|
||||
}
|
||||
|
||||
func serveHoldCtl(conn net.Conn, clear func(stack, who string) (string, error), logger *log.Logger) {
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
answer := func(r holdCtlResponse) { _ = json.NewEncoder(conn).Encode(r) }
|
||||
if ok, why := holdCtlPeerAllowed(conn); !ok {
|
||||
if logger != nil {
|
||||
logger.Printf("[WARN] [hold] local channel: a request was refused — %s", why)
|
||||
}
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "refused: " + why})
|
||||
return
|
||||
}
|
||||
line, err := bufio.NewReader(io.LimitReader(conn, holdCtlMaxRequest)).ReadBytes('\n')
|
||||
if err != nil && len(line) == 0 {
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "no request was read"})
|
||||
return
|
||||
}
|
||||
var req holdCtlRequest
|
||||
if err := json.Unmarshal(line, &req); err != nil || req.Op != holdCtlOpClear {
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: "unknown request — this channel only clears a hold"})
|
||||
return
|
||||
}
|
||||
msg, err := clear(req.Stack, "the operator on the box (--clear-restore-hold)")
|
||||
switch {
|
||||
case err == nil:
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeDone, Message: msg})
|
||||
case errors.Is(err, report.ErrActionRefused):
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeRefused, Message: strings.TrimPrefix(err.Error(), report.ErrActionRefused.Error()+": ")})
|
||||
default:
|
||||
answer(holdCtlResponse{Outcome: report.OutcomeFailed, Message: err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
// askRunningControllerToClear sends one clear request. reached=false: no controller answered at path (no socket, or
|
||||
// a socket nobody listens on) — nothing was asked, so the caller may edit settings.json itself. reached=true with an
|
||||
// error: the request may have been applied; the caller must not edit the file.
|
||||
func askRunningControllerToClear(path, stack string) (resp holdCtlResponse, reached bool, err error) {
|
||||
conn, err := net.DialTimeout("unix", path, 3*time.Second)
|
||||
if err != nil {
|
||||
return holdCtlResponse{}, false, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(60 * time.Second))
|
||||
if err := json.NewEncoder(conn).Encode(holdCtlRequest{Op: holdCtlOpClear, Stack: stack}); err != nil {
|
||||
return holdCtlResponse{}, true, err
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(conn, 64*1024)).Decode(&resp); err != nil {
|
||||
return holdCtlResponse{}, true, err
|
||||
}
|
||||
return resp, true, nil
|
||||
}
|
||||
|
||||
// clearRestoreHoldCLI is the box command `--clear-restore-hold <app>` (R-379, R-1343). It asks the running
|
||||
// controller first; only when none answers does it edit settings.json itself, and it says which one happened.
|
||||
// Returns the process exit code: 0 cleared, 2 refused (nothing changed), 1 failed or unknown.
|
||||
func clearRestoreHoldCLI(dataDir, stack string, stdout, stderr io.Writer) int {
|
||||
path := holdCtlPath(dataDir)
|
||||
resp, reached, err := askRunningControllerToClear(path, stack)
|
||||
if reached {
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "the running controller was reached, but its answer was lost: %v\n", err)
|
||||
fmt.Fprintf(stderr, " The hold may or may not be cleared. Check with: felhom-controller --restore-holds\n")
|
||||
fmt.Fprintf(stderr, " settings.json was NOT edited by this command.\n")
|
||||
return 1
|
||||
}
|
||||
switch resp.Outcome {
|
||||
case report.OutcomeDone:
|
||||
fmt.Fprintf(stdout, "%s.\n", resp.Message)
|
||||
fmt.Fprintf(stdout, " Cleared by the RUNNING controller (memory and settings.json together) — no restart is needed.\n")
|
||||
return 0
|
||||
case report.OutcomeRefused:
|
||||
fmt.Fprintf(stderr, "%s\n", resp.Message)
|
||||
return 2
|
||||
default:
|
||||
fmt.Fprintf(stderr, "the running controller could not clear the hold: %s\n", resp.Message)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
// No controller answered: it is not running (or is older than this channel). The file is the only copy.
|
||||
fmt.Fprintf(stdout, "No running controller answered on %s (%v).\n", path, err)
|
||||
fmt.Fprintf(stdout, " Editing settings.json directly instead.\n")
|
||||
if !backup.IsValidHoldStack(stack) {
|
||||
fmt.Fprintf(stderr, "%q is not a valid app name — nothing was changed\n", stack)
|
||||
return 2
|
||||
}
|
||||
lg := log.New(stderr, "", 0)
|
||||
sett, lerr := settings.Load(filepath.Join(dataDir, "settings.json"), lg)
|
||||
if lerr != nil {
|
||||
fmt.Fprintf(stderr, "restore-hold: loading settings: %v\n", lerr)
|
||||
return 1
|
||||
}
|
||||
cleared, cerr := sett.ClearRestoreHold(stack)
|
||||
if cerr != nil {
|
||||
fmt.Fprintf(stderr, "restore-hold: clearing %s: %v\n", stack, cerr)
|
||||
return 1
|
||||
}
|
||||
if !cleared {
|
||||
// "there was nothing to clear" is not success — a silent 0 here would let an operator believe they had
|
||||
// unblocked an app whose name they mistyped.
|
||||
fmt.Fprintf(stderr, "no restore hold in force for %q — nothing was changed\n", stack)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintf(stdout, "restore hold cleared for %s in settings.json (no controller was running to ask).\n", stack)
|
||||
fmt.Fprintf(stdout, " The controller reads it when it starts. If a controller IS running in this guest after all, restart it NOW,\n")
|
||||
fmt.Fprintf(stdout, " INSIDE THE GUEST — from the Proxmox host: pct exec <vmid> -- systemctl restart felhom-controller-bootstrap.service\n")
|
||||
fmt.Fprintf(stdout, " — or its next settings save writes the hold back (R-1343).\n")
|
||||
return 0
|
||||
}
|
||||
|
||||
// holdClearCommandHint is the exact guest command, for the help text and the operator's mail.
|
||||
const holdClearCommandHint = "pct exec <vmid> -- docker exec felhom-controller felhom-controller --clear-restore-hold <app>"
|
||||
@@ -0,0 +1,516 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/holdreport"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// D64 = B (operator ruling 2026-10-29 12:10) + R-1343: the operator clears a hold — from the hub (clear_hold) or with
|
||||
// the box command — IN THE RUNNING CONTROLLER, so the hold is gone from memory and from settings.json together and no
|
||||
// later save brings it back.
|
||||
//
|
||||
// RED-PROOFS (2026-10-29):
|
||||
// - TestR1343_TheBoxCommandClearsThroughTheRunningController: with clearRestoreHoldCLI's socket call skipped (the
|
||||
// old second-process edit of settings.json), it fails on „the hold came back after a later save of the running
|
||||
// controller" — the R-1343 measurement, in a test;
|
||||
// - TestD64_ClearPutBackPendingDropsTheMarkerEntry: with the dropFromMarker call removed from
|
||||
// ClearHoldByOperator, it fails on „a later controller start poured the old copy back";
|
||||
// - TestD64_ClearHoldViaTheHub: with the forgetUpdateHeld call removed from clearHoldNow, it fails on „the
|
||||
// update's held flag was not forgotten".
|
||||
// - TestD64_ClearHoldViaTheHub + TestD64_ClearHoldStaysGoneAfterALaterSave: with ClearHoldByOperator clearing a
|
||||
// SECOND Settings loaded from disk (the old CLI's shape) instead of the running one, they fail on „the running
|
||||
// controller still holds the app" / „the hold came back after a later save of the running controller".
|
||||
|
||||
// d64Rig is a running controller's hold side: its Settings, the app-stop guard wired to them, and a log.
|
||||
type d64Rig struct {
|
||||
dir string
|
||||
sett *settings.Settings
|
||||
guard *backup.AppStopGuard
|
||||
logBuf *syncBuffer
|
||||
logger *log.Logger
|
||||
forgot []string
|
||||
holdDeps holdClearDeps
|
||||
}
|
||||
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.Write(p)
|
||||
}
|
||||
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
|
||||
|
||||
func newD64Rig(t *testing.T) *d64Rig {
|
||||
t.Helper()
|
||||
r := &d64Rig{dir: t.TempDir(), logBuf: &syncBuffer{}}
|
||||
r.logger = log.New(r.logBuf, "", 0)
|
||||
sett, err := settings.Load(filepath.Join(r.dir, "settings.json"), r.logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.sett = sett
|
||||
r.guard = backup.NewAppStopGuard(filepath.Join(r.dir, "appstop-state.json"), r.logger)
|
||||
r.guard.SetHolds(sett)
|
||||
r.holdDeps = holdClearDeps{
|
||||
clear: r.guard.ClearHoldByOperator,
|
||||
forgetUpdateHeld: func(s string) bool { r.forgot = append(r.forgot, s); return true },
|
||||
logger: r.logger,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *d64Rig) hold(t *testing.T, h settings.RestoreHold) {
|
||||
t.Helper()
|
||||
if h.At == "" {
|
||||
h.At = "2026-10-29T08:18:00Z"
|
||||
}
|
||||
if err := r.sett.SetRestoreHold(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *d64Rig) onDisk(t *testing.T, stack string) bool {
|
||||
t.Helper()
|
||||
s, err := settings.Load(filepath.Join(r.dir, "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, held := s.GetRestoreHold(stack)
|
||||
return held
|
||||
}
|
||||
|
||||
func (r *d64Rig) actions() *report.OperatorActions {
|
||||
return report.NewOperatorActions(context.Background(), operatorActionHandlers(nil, nil, r.holdDeps, nil, r.logger), r.logger)
|
||||
}
|
||||
|
||||
func onlyResult(t *testing.T, o *report.OperatorActions, id int64) report.OperatorActionResult {
|
||||
t.Helper()
|
||||
for _, x := range o.Results() {
|
||||
if x.ID == id {
|
||||
return x
|
||||
}
|
||||
}
|
||||
t.Fatalf("no result for #%d: %+v", id, o.Results())
|
||||
return report.OperatorActionResult{}
|
||||
}
|
||||
|
||||
// The hub's clear_hold: the hold is gone (memory and disk), the household's held sentence is gone, the update's
|
||||
// held flag is forgotten, the box logs who did it, and the result names what was cleared.
|
||||
func TestD64_ClearHoldViaTheHub(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "paperless", ReplayError: "ERROR: relation exists", RollbackErr: "psql: refused"})
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = r.dir
|
||||
mgr := backup.NewManager(cfg, r.sett, r.logger)
|
||||
if held, why := mgr.RestoreHoldForLang("paperless", "hu"); !held || why == "" {
|
||||
t.Fatal("fixture: the household must read a held sentence before the clear")
|
||||
}
|
||||
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 12, Action: report.OpClearHold, Arg: "paperless"}})
|
||||
res := onlyResult(t, o, 12)
|
||||
if res.Outcome != report.OutcomeDone {
|
||||
t.Fatalf("result %+v", res)
|
||||
}
|
||||
for _, want := range []string{"the hold on paperless is cleared", "held after a failed restore whose rollback also failed", "NOT started by this"} {
|
||||
if !strings.Contains(res.Message, want) {
|
||||
t.Errorf("result message %q lacks %q", res.Message, want)
|
||||
}
|
||||
}
|
||||
if _, held := r.sett.GetRestoreHold("paperless"); held {
|
||||
t.Fatal("the running controller still holds the app")
|
||||
}
|
||||
if r.onDisk(t, "paperless") {
|
||||
t.Fatal("settings.json still holds the app")
|
||||
}
|
||||
if held, why := mgr.RestoreHoldForLang("paperless", "hu"); held || why != "" {
|
||||
t.Fatalf("the household still reads a held sentence: %q", why)
|
||||
}
|
||||
if len(r.forgot) != 1 || r.forgot[0] != "paperless" {
|
||||
t.Fatalf("the update's held flag was not forgotten: %v", r.forgot)
|
||||
}
|
||||
if lg := r.logBuf.String(); !strings.Contains(lg, "paperless: hold CLEARED by the operator via the hub, action #12") {
|
||||
t.Fatalf("the box's log must say who cleared it:\n%s", lg)
|
||||
}
|
||||
}
|
||||
|
||||
// Refusals change nothing: an app nobody ever held, a second press on an app already cleared, a bad name.
|
||||
func TestD64_ClearHoldRefusals(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "kimai", Reason: settings.HoldReasonUpdateFailed})
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpClearHold, Arg: "nosuchapp"}})
|
||||
if res := onlyResult(t, o, 1); res.Outcome != report.OutcomeRefused || !strings.Contains(res.Message, "nosuchapp has no hold") {
|
||||
t.Fatalf("unknown app: %+v", res)
|
||||
}
|
||||
if _, held := r.sett.GetRestoreHold("kimai"); !held || !r.onDisk(t, "kimai") {
|
||||
t.Fatal("a refusal changed another app's hold")
|
||||
}
|
||||
if len(r.forgot) != 0 {
|
||||
t.Fatalf("a refusal touched the update state: %v", r.forgot)
|
||||
}
|
||||
o.Reconcile([]report.OperatorAction{{ID: 2, Action: report.OpClearHold, Arg: "kimai"}})
|
||||
o.Reconcile([]report.OperatorAction{{ID: 3, Action: report.OpClearHold, Arg: "kimai"}})
|
||||
if res := onlyResult(t, o, 3); res.Outcome != report.OutcomeRefused || !strings.Contains(res.Message, "kimai has no hold") {
|
||||
t.Fatalf("not held (second press): %+v", res)
|
||||
}
|
||||
if _, err := clearHoldNow(r.holdDeps, "../kimai", "test"); err == nil || !strings.Contains(err.Error(), "not a valid app name") {
|
||||
t.Fatalf("a bad name must be refused by the door itself too: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1343's order: clear, then ANOTHER component saves settings (the hold reporter, a hub report), then the file is
|
||||
// read back — the hold must stay gone, in memory and on disk.
|
||||
func TestD64_ClearHoldStaysGoneAfterALaterSave(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed})
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 5, Action: report.OpClearHold, Arg: "radicale"}})
|
||||
if res := onlyResult(t, o, 5); res.Outcome != report.OutcomeDone {
|
||||
t.Fatalf("result %+v", res)
|
||||
}
|
||||
if err := r.sett.SetReportedHolds(map[string]string{"other": "x|y"}); err != nil { // another component's save
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.onDisk(t, "radicale") {
|
||||
t.Fatal("the hold came back after a later save of the running controller")
|
||||
}
|
||||
if _, held := r.sett.GetRestoreHold("radicale"); held {
|
||||
t.Fatal("the running controller holds the app again")
|
||||
}
|
||||
}
|
||||
|
||||
// The same order through the BOX COMMAND: it reaches the running controller over the socket; no restart.
|
||||
func TestR1343_TheBoxCommandClearsThroughTheRunningController(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed})
|
||||
ln, err := listenHoldCtl(holdCtlPath(r.dir), func(stack, who string) (string, error) { return clearHoldNow(r.holdDeps, stack, who) }, r.logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
if code := clearRestoreHoldCLI(r.dir, "radicale", &out, &errb); code != 0 {
|
||||
t.Fatalf("exit %d, stdout %q, stderr %q", code, out.String(), errb.String())
|
||||
}
|
||||
if err := r.sett.SetReportedHolds(map[string]string{"other": "x|y"}); err != nil { // the running controller's next save
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.onDisk(t, "radicale") {
|
||||
t.Fatal("the hold came back after a later save of the running controller")
|
||||
}
|
||||
if !strings.Contains(out.String(), "Cleared by the RUNNING controller") || !strings.Contains(out.String(), "no restart is needed") {
|
||||
t.Fatalf("the command must say it went through the running controller and needs no restart:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(r.logBuf.String(), "hold CLEARED by the operator on the box (--clear-restore-hold)") {
|
||||
t.Fatalf("the box's log must say who cleared it:\n%s", r.logBuf.String())
|
||||
}
|
||||
|
||||
// A second run: refused through the socket, exit 2, nothing edited.
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
if code := clearRestoreHoldCLI(r.dir, "radicale", &out, &errb); code != 2 || !strings.Contains(errb.String(), "radicale has no hold") {
|
||||
t.Fatalf("not held: exit %d, stderr %q", code, errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
// No controller runs (no socket, or a socket nobody listens on): the command edits settings.json and says so plainly,
|
||||
// naming the guest restart for the case a controller runs after all.
|
||||
func TestR1343_TheBoxCommandFallsBackWhenNoControllerRuns(t *testing.T) {
|
||||
for _, stale := range []bool{false, true} {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "kimai", Reason: settings.HoldReasonUpdateFailed})
|
||||
if stale { // a socket left by a controller that died
|
||||
ln, err := listenHoldCtl(holdCtlPath(r.dir), func(string, string) (string, error) { return "", nil }, r.logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ln.(*net.UnixListener).SetUnlinkOnClose(false)
|
||||
ln.Close()
|
||||
}
|
||||
var out, errb bytes.Buffer
|
||||
if code := clearRestoreHoldCLI(r.dir, "kimai", &out, &errb); code != 0 {
|
||||
t.Fatalf("stale=%v: exit %d, stderr %q", stale, code, errb.String())
|
||||
}
|
||||
for _, want := range []string{"No running controller answered", "Editing settings.json directly", "INSIDE THE GUEST", "pct exec <vmid> -- systemctl restart felhom-controller-bootstrap.service"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("stale=%v: output lacks %q:\n%s", stale, want, out.String())
|
||||
}
|
||||
}
|
||||
if r.onDisk(t, "kimai") {
|
||||
t.Fatalf("stale=%v: settings.json still holds the app", stale)
|
||||
}
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
if code := clearRestoreHoldCLI(r.dir, "kimai", &out, &errb); code != 2 {
|
||||
t.Fatalf("stale=%v: not held must exit 2, got %d", stale, code)
|
||||
}
|
||||
if code := clearRestoreHoldCLI(r.dir, "../x", &out, &errb); code != 2 {
|
||||
t.Fatalf("stale=%v: a bad name must exit 2, got %d", stale, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A put-back WAITING for its drive is also in the app-stop marker: clearing the hold must drop it there too, or the
|
||||
// next controller start with the drive bound would pour the old copy over what the app wrote after the clear. The
|
||||
// copy itself stays on disk (nothing is deleted).
|
||||
func TestD64_ClearPutBackPendingDropsTheMarkerEntry(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
undo := filepath.Join(r.dir, "drive", "felhom-data", "backups", "restore-undo", "immich", "20261029T080000Z")
|
||||
if err := os.MkdirAll(undo, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(undo, "undo.json"), []byte(`{"stack":"immich"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.guard.Begin("unit-restore:immich", backup.ReasonUnitRestore, []string{"immich"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.guard.SetUndo("immich", undo); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.hold(t, settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonPutBackPending, SafetyDump: undo})
|
||||
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 3, Action: report.OpClearHold, Arg: "immich"}})
|
||||
res := onlyResult(t, o, 3)
|
||||
if res.Outcome != report.OutcomeDone || !strings.Contains(res.Message, "waiting put-back is dropped") || !strings.Contains(res.Message, undo) {
|
||||
t.Fatalf("result %+v", res)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(undo, "undo.json")); err != nil {
|
||||
t.Fatal("the copy must stay on disk")
|
||||
}
|
||||
// The consequence: a later controller start with the drive bound pours nothing back.
|
||||
poured := &recordingPutBacker{}
|
||||
g2 := backup.NewAppStopGuard(filepath.Join(r.dir, "appstop-state.json"), r.logger)
|
||||
g2.SetHolds(r.sett)
|
||||
g2.SetPutBacker(poured)
|
||||
g2.SetStarter(nopStarter{})
|
||||
g2.SetUndoReachable(func(string, string) bool { return true })
|
||||
g2.Recover()
|
||||
if len(poured.stacks) != 0 {
|
||||
t.Fatalf("a later controller start poured the old copy back over the app: %v", poured.stacks)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingPutBacker struct{ stacks []string }
|
||||
|
||||
func (p *recordingPutBacker) PutBackInterruptedRestore(stack, _ string) error {
|
||||
p.stacks = append(p.stacks, stack)
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopStarter struct{}
|
||||
|
||||
func (nopStarter) StartStack(string) error { return nil }
|
||||
|
||||
// The hold reporter's next tick tells the hub the hold is gone (it reads the same settings record; nothing cached).
|
||||
func TestD64_TheHoldReporterTellsTheHubAfterTheClear(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "kimai", Reason: settings.HoldReasonUpdateFailed})
|
||||
var events []string
|
||||
rep := holdreport.New(r.sett, func(et, _, _ string, _ interface{}) { events = append(events, et) }, r.logger)
|
||||
rep.Tick(timeAt(0))
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpClearHold, Arg: "kimai"}})
|
||||
rep.Tick(timeAt(60))
|
||||
if strings.Join(events, ",") != holdreport.EventHeld+","+holdreport.EventCleared {
|
||||
t.Fatalf("events %v — want app_held then app_hold_cleared", events)
|
||||
}
|
||||
}
|
||||
|
||||
// „Not reachable from outside": the channel is a unix socket (no TCP anywhere in the file), 0600 in a 0700 directory
|
||||
// — even when the directory existed with a looser mode — and main opens it once, under the data dir.
|
||||
func TestHoldCtl_NotReachableFromOutside(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, holdCtlDir), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ln, err := listenHoldCtl(holdCtlPath(dir), func(string, string) (string, error) { return "", nil }, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
if n := ln.Addr().Network(); n != "unix" {
|
||||
t.Fatalf("the channel listens on %q, want unix", n)
|
||||
}
|
||||
fi, err := os.Lstat(holdCtlPath(dir))
|
||||
if err != nil || fi.Mode()&os.ModeSocket == 0 || fi.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("socket mode %v (err %v), want a socket with 0600", fi.Mode(), err)
|
||||
}
|
||||
di, _ := os.Stat(filepath.Join(dir, holdCtlDir))
|
||||
if di.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("socket dir mode %v, want 0700", di.Mode().Perm())
|
||||
}
|
||||
|
||||
// No TCP and no HTTP in the channel's file: every net.Listen/Dial names "unix".
|
||||
f, err := parser.ParseFile(token.NewFileSet(), "holdclear.go", nil, parser.ImportsOnly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, im := range f.Imports {
|
||||
if p, _ := strconv.Unquote(im.Path.Value); strings.HasPrefix(p, "net/http") {
|
||||
t.Fatalf("holdclear.go imports %s — the channel must not be HTTP", p)
|
||||
}
|
||||
}
|
||||
full, err := parser.ParseFile(token.NewFileSet(), "holdclear.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nets := 0
|
||||
ast.Inspect(full, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "net" && len(call.Args) > 0 {
|
||||
nets++
|
||||
if lit, ok := call.Args[0].(*ast.BasicLit); !ok || lit.Value != `"unix"` {
|
||||
t.Errorf("net.%s is called with a network other than \"unix\"", sel.Sel.Name)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if nets < 2 {
|
||||
t.Fatalf("expected the listen and the dial in holdclear.go, found %d net calls", nets)
|
||||
}
|
||||
|
||||
// main opens it exactly once, at holdCtlPath(cfg.Paths.DataDir).
|
||||
mf, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opened := 0
|
||||
ast.Inspect(mf, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "listenHoldCtl" {
|
||||
opened++
|
||||
arg, ok := call.Args[0].(*ast.CallExpr)
|
||||
if fn, ok2 := arg.Fun.(*ast.Ident); !ok || !ok2 || fn.Name != "holdCtlPath" {
|
||||
t.Error("main must open the channel at holdCtlPath(<data dir>)")
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if opened != 1 {
|
||||
t.Fatalf("main.go opens the channel %d time(s), want 1", opened)
|
||||
}
|
||||
}
|
||||
|
||||
// The socket answers only its one request; anything else is refused and calls nothing.
|
||||
func TestHoldCtl_UnknownRequestIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
called := 0
|
||||
ln, err := listenHoldCtl(holdCtlPath(dir), func(string, string) (string, error) { called++; return "x", nil }, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
conn, err := net.Dial("unix", holdCtlPath(dir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := conn.Write([]byte(`{"op":"set_hold","stack":"x"}` + "\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var resp holdCtlResponse
|
||||
if err := json.NewDecoder(conn).Decode(&resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.Outcome != report.OutcomeRefused || called != 0 {
|
||||
t.Fatalf("response %+v, calls %d — want refused and nothing called", resp, called)
|
||||
}
|
||||
}
|
||||
|
||||
// Something at the socket's path that is NOT a socket is refused, never removed.
|
||||
func TestHoldCtl_ANonSocketFileIsNotReplaced(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := holdCtlPath(dir)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte("keep me"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listenHoldCtl(p, func(string, string) (string, error) { return "", nil }, nil); err == nil {
|
||||
t.Fatal("a regular file at the socket path must be refused")
|
||||
}
|
||||
if b, _ := os.ReadFile(p); string(b) != "keep me" {
|
||||
t.Fatal("the file at the socket path was changed")
|
||||
}
|
||||
}
|
||||
|
||||
// main wires every piece once (the seam-built-but-never-wired trap): the clear door into the operator actions, the
|
||||
// R-1340 text into the hold notification, the R-1341 gate into the OOM loop, and the box command.
|
||||
func TestD64_MainWiresTheNewPieces(t *testing.T) {
|
||||
f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := map[string]int{}
|
||||
var opArgs []ast.Expr
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch fn := call.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := fn.X.(*ast.Ident); ok {
|
||||
found[x.Name+"."+fn.Sel.Name]++
|
||||
}
|
||||
case *ast.Ident:
|
||||
found[fn.Name]++
|
||||
if fn.Name == "operatorActionHandlers" {
|
||||
opArgs = call.Args
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
for _, want := range []string{"heldStoppedOperatorText", "heldHoldFromNotify", "oomLog.fresh", "clearRestoreHoldCLI", "listenHoldCtl", "clearHoldNow"} {
|
||||
if found[want] != 1 {
|
||||
t.Errorf("main.go calls %s %d time(s), want exactly 1", want, found[want])
|
||||
}
|
||||
}
|
||||
if len(opArgs) < 3 {
|
||||
t.Fatal("operatorActionHandlers call not found")
|
||||
}
|
||||
if id, ok := opArgs[2].(*ast.Ident); !ok || id.Name != "holdDeps" {
|
||||
t.Error("main must hand holdDeps (the running controller's clear door) to operatorActionHandlers")
|
||||
}
|
||||
}
|
||||
|
||||
func timeAt(sec int) time.Time { return time.Date(2026, 10, 29, 12, 0, sec, 0, time.UTC) }
|
||||
@@ -0,0 +1,34 @@
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// holdCtlPeerAllowed (R-1343) admits a peer on the hold socket only when its uid equals this process's (SO_PEERCRED).
|
||||
// The socket's 0600 mode in a 0700 directory already says so; this is the second lock on the same door.
|
||||
func holdCtlPeerAllowed(conn net.Conn) (bool, string) {
|
||||
uc, ok := conn.(*net.UnixConn)
|
||||
if !ok {
|
||||
return false, "not a unix-socket peer"
|
||||
}
|
||||
raw, err := uc.SyscallConn()
|
||||
if err != nil {
|
||||
return false, "the peer's identity could not be read"
|
||||
}
|
||||
var cred *syscall.Ucred
|
||||
var cerr error
|
||||
if err := raw.Control(func(fd uintptr) {
|
||||
cred, cerr = syscall.GetsockoptUcred(int(fd), syscall.SOL_SOCKET, syscall.SO_PEERCRED)
|
||||
}); err != nil || cerr != nil || cred == nil {
|
||||
return false, "the peer's identity could not be read"
|
||||
}
|
||||
if int(cred.Uid) != os.Geteuid() {
|
||||
return false, fmt.Sprintf("the peer's uid %d is not this controller's", cred.Uid)
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !linux
|
||||
|
||||
package main
|
||||
|
||||
import "net"
|
||||
|
||||
// holdCtlPeerAllowed: the peer check needs SO_PEERCRED (Linux, where the controller runs). Elsewhere the socket's
|
||||
// 0600 mode in a 0700 directory is the only lock.
|
||||
func holdCtlPeerAllowed(conn net.Conn) (bool, string) {
|
||||
if _, ok := conn.(*net.UnixConn); !ok {
|
||||
return false, "not a unix-socket peer"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── R-1340: the operator's „HELD STOPPED" mail names the hold's REAL cause ────────────────────────────────────────
|
||||
//
|
||||
// Until 2026-10-29 the restore-hold mail told one story for every restore_mixed hold — R-893 option C's „its off-site
|
||||
// database restore failed. The database was rolled back …, but its files and named volumes are the snapshot's" —
|
||||
// written when option C was the only mixed hold. Measured on Tester 1 (D63's H5 proof): Radicale, which has no
|
||||
// database, whose put-back stopped at „the app could not be stopped" with nothing moved, got exactly that text.
|
||||
// Support read a wrong account of the box.
|
||||
//
|
||||
// The text is now built from the HOLD RECORD the writer persisted just before the mail (its reason and the R-1340
|
||||
// fields: which restore, which branch, which step failed, whether the failure was a database replay, where the copy
|
||||
// is kept). It never says „database" unless the restore's failure WAS a database replay, and never „off-site" for a
|
||||
// restore from the app's own backup. One sentence per reason; pinned by TestR1340_* (holdmail_test.go).
|
||||
|
||||
// heldStoppedOperatorText is the operator mail for a held app: the message, its leg (the hub reads the leg, so the
|
||||
// two existing ones are kept: restore-hold for a restore hold, restore-hold-mixed for a mixed one) and a short reason.
|
||||
func heldStoppedOperatorText(h settings.RestoreHold) (msg, leg, reason string) {
|
||||
stack := h.Stack
|
||||
const tail = " The app will not start from any path until the hold is cleared (the hub's Clear hold button, or on the box: " + holdClearCommandHint + ")."
|
||||
kind := restoreKindPhrase(h.RestoreKind)
|
||||
switch h.Reason {
|
||||
case settings.HoldReasonUnhealthyStop:
|
||||
what := "a crash loop"
|
||||
if h.UnhealthyKind == "oom_storm" {
|
||||
what = "an out-of-memory storm"
|
||||
}
|
||||
trip := ""
|
||||
if h.Trip >= 2 {
|
||||
trip = fmt.Sprintf(" This is stop %d within 24 hours.", h.Trip)
|
||||
}
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED: the box stopped it for %s at %s.%s No restore or update was involved and no data was touched. "+
|
||||
"The household's Start gives it one more try; clearing the hold lifts it too.", stack, what, h.At, trip)
|
||||
return msg, "unhealthy-stop", "stopped for " + what
|
||||
case settings.HoldReasonUpdateFailed:
|
||||
undo := ""
|
||||
switch h.UndoState {
|
||||
case "untouched":
|
||||
undo = " The automatic undo failed; the data was not touched."
|
||||
case "half":
|
||||
undo = " The automatic undo failed half-way; the data may be partly the new version's."
|
||||
case "not_started":
|
||||
undo = " The automatic undo did not start."
|
||||
}
|
||||
copyLine := " Restoring it from its backup starts it."
|
||||
if h.NoWholeCopy {
|
||||
copyLine = " No copy on this box brings it back whole — it needs support."
|
||||
} else if h.CopyDate != "" {
|
||||
copyLine = fmt.Sprintf(" Restoring it from its backup (tier %d, %s) starts it.", h.CopyTier, h.CopyDate)
|
||||
}
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED: its update at %s did not come up healthy.%s%s", stack, h.At, undo, copyLine) + tail
|
||||
return msg, "update-hold", "update failed"
|
||||
case settings.HoldReasonPutBackPending:
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED: a restore was cut after its copy of the live data was taken, and that copy (%s) is on a drive that is not bound. "+
|
||||
"The copy is poured back and the app started when the drive is bound. Clearing the hold DROPS that put-back; the copy stays on disk.", stack, orNone(h.SafetyDump))
|
||||
return msg, "putback-pending", "put-back waits for its drive"
|
||||
case settings.HoldReasonRestoreMixed:
|
||||
return mixedHoldText(h, kind, tail)
|
||||
default: // HoldReasonRestoreFailed: a database replay failed AND its rollback failed (holdAppAfterFailedRollback)
|
||||
what := "its " + kind + " failed"
|
||||
if h.DBReplay || h.RestoreKind == "" { // every restore hold before R-1340 was written by the database branch
|
||||
what = "its " + kind + " failed at the database replay AND the rollback to the pre-restore database copy also failed — the database may be half-written"
|
||||
}
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED: %s.", stack, what) + tail +
|
||||
fmt.Sprintf(" Replay error: %s. Rollback error: %s", orNone(h.ReplayError), orNone(h.RollbackErr))
|
||||
reason = h.RollbackErr
|
||||
if reason == "" {
|
||||
reason = "rollback failed"
|
||||
}
|
||||
return msg, "restore-hold", reason
|
||||
}
|
||||
}
|
||||
|
||||
func mixedHoldText(h settings.RestoreHold, kind, tail string) (msg, leg, reason string) {
|
||||
stack := h.Stack
|
||||
leg = "restore-hold-mixed"
|
||||
failedAt := ""
|
||||
if h.DBReplay {
|
||||
failedAt = " at the database replay"
|
||||
}
|
||||
copyLine := " There is no copy of the live volumes."
|
||||
if h.SafetyDump != "" {
|
||||
copyLine = " The copy of the live data is kept at " + h.SafetyDump + "."
|
||||
}
|
||||
switch h.MixedCause {
|
||||
case settings.MixedCauseDefinitionWrite:
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED for support: its %s failed while writing the backup's app definition, before any data was touched, "+
|
||||
"and %s. The app's data is the live data; the definition on disk may be the backup's.", stack, kind, orNone(h.FailedStep)) + tail +
|
||||
" Error: " + orNone(h.ReplayError)
|
||||
return msg, leg, "definition write failed; writing the live definition back failed too"
|
||||
case settings.MixedCausePutBack:
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED for support: its %s failed%s, and putting the live data back FAILED — %s. "+
|
||||
"The app may be on a mix of the backup's and the live data.%s", stack, kind, failedAt, orNone(h.FailedStep), copyLine) + tail +
|
||||
" Restore error: " + orNone(h.ReplayError)
|
||||
return msg, leg, "put-back failed: " + firstClause(h.FailedStep)
|
||||
case settings.MixedCauseInterruptedPutBack:
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED for support: a restore cut by a stop could not be put back at the next start — %s. "+
|
||||
"The app may be on a mix of the backup's and the live data.%s", stack, orNone(h.FailedStep), copyLine) + tail
|
||||
return msg, leg, "interrupted restore: put-back failed"
|
||||
}
|
||||
// A mixed hold written before R-1340 carries no cause: name only what the record says, and assume nothing.
|
||||
msg = fmt.Sprintf("App %q is HELD STOPPED for support: a restore stopped half-way (set %s); the app may be on a mix of the backup's and the live data. "+
|
||||
"The hold record says: %s.", stack, h.At, orNone(h.ReplayError)) + tail
|
||||
return msg, leg, "restore stopped half-way"
|
||||
}
|
||||
|
||||
func restoreKindPhrase(k string) string {
|
||||
switch k {
|
||||
case settings.RestoreKindLocal:
|
||||
return "restore from its own backup"
|
||||
case settings.RestoreKindOffsite:
|
||||
return "off-site restore"
|
||||
}
|
||||
return "restore"
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "(none recorded)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// firstClause is a step's text up to its error (the short reason field).
|
||||
func firstClause(s string) string {
|
||||
if i := strings.Index(s, ":"); i > 0 {
|
||||
return s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// heldHoldFromNotify is the hold the notification describes: the record the writer persisted just before it, or —
|
||||
// when that write failed — one rebuilt from the errors handed over (the pre-R-1340 rule: no rollback error = mixed).
|
||||
func heldHoldFromNotify(get func(string) (settings.RestoreHold, bool), stack string, replayErr, rollbackErr error) settings.RestoreHold {
|
||||
if get != nil {
|
||||
if h, ok := get(stack); ok && (h.Reason == settings.HoldReasonRestoreFailed || h.Reason == settings.HoldReasonRestoreMixed) {
|
||||
return h
|
||||
}
|
||||
}
|
||||
h := settings.RestoreHold{Stack: stack, Reason: settings.HoldReasonRestoreMixed}
|
||||
if rollbackErr != nil {
|
||||
h.Reason, h.RollbackErr = settings.HoldReasonRestoreFailed, rollbackErr.Error()
|
||||
}
|
||||
if replayErr != nil {
|
||||
h.ReplayError = replayErr.Error()
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// R-1340: the operator's „HELD STOPPED" mail names the hold's real cause — one test per reason. Each asserts the words
|
||||
// that MUST be there and the words that must NOT (the option-C database story was the bug).
|
||||
//
|
||||
// RED-PROOF (2026-10-29): with mixedHoldText's body replaced by the pre-R-1340 sentence („its off-site database restore
|
||||
// failed. The database was rolled back to the pre-restore copy and the live definition written back, but its files
|
||||
// and named volumes are the snapshot's"), TestR1340_PutBackWithoutADatabase, _DefinitionWrite, _LocalRestore and
|
||||
// _InterruptedPutBack fail on „database" / „off-site" being present and their own step missing.
|
||||
|
||||
const optionCStory = "files and named volumes are the snapshot's"
|
||||
|
||||
func mustHave(t *testing.T, msg string, want ...string) {
|
||||
t.Helper()
|
||||
for _, w := range want {
|
||||
if !strings.Contains(msg, w) {
|
||||
t.Errorf("the mail must say %q:\n%s", w, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustNotHave(t *testing.T, msg string, bad ...string) {
|
||||
t.Helper()
|
||||
for _, b := range bad {
|
||||
if strings.Contains(strings.ToLower(msg), strings.ToLower(b)) {
|
||||
t.Errorf("the mail must NOT say %q:\n%s", b, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1340_UnhealthyStop(t *testing.T) {
|
||||
msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "romm", At: "2026-10-29T08:00:00Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"})
|
||||
mustHave(t, msg, `"romm"`, "crash loop", "no data was touched", "Start gives it one more try")
|
||||
mustNotHave(t, msg, "database", "restore failed", optionCStory)
|
||||
if leg != "unhealthy-stop" {
|
||||
t.Errorf("leg %q", leg)
|
||||
}
|
||||
oom, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "romm", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "oom_storm", Trip: 2})
|
||||
mustHave(t, oom, "out-of-memory storm", "stop 2 within 24 hours")
|
||||
}
|
||||
|
||||
func TestR1340_UpdateFailed(t *testing.T) {
|
||||
msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "kimai", At: "2026-10-29T03:10:00Z", Reason: settings.HoldReasonUpdateFailed,
|
||||
UndoState: "half", CopyTier: 1, CopyDate: "2026-10-29T02:00:00Z"})
|
||||
mustHave(t, msg, "update at 2026-10-29T03:10:00Z did not come up healthy", "undo failed half-way", "tier 1, 2026-10-29T02:00:00Z", "--clear-restore-hold")
|
||||
mustNotHave(t, msg, "database", optionCStory)
|
||||
if leg != "update-hold" {
|
||||
t.Errorf("leg %q", leg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1340_RestoreFailedWithADatabase(t *testing.T) {
|
||||
msg, leg, reason := heldStoppedOperatorText(settings.RestoreHold{Stack: "paperless", Reason: settings.HoldReasonRestoreFailed, RestoreKind: settings.RestoreKindOffsite,
|
||||
DBReplay: true, ReplayError: "ERROR: relation exists", RollbackErr: "psql: connection refused"})
|
||||
mustHave(t, msg, "off-site restore failed at the database replay", "rollback to the pre-restore database copy also failed", "ERROR: relation exists", "psql: connection refused")
|
||||
mustNotHave(t, msg, optionCStory)
|
||||
if leg != "restore-hold" || reason != "psql: connection refused" {
|
||||
t.Errorf("leg %q reason %q", leg, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// The measured case: Radicale has no database; the volume could not be removed and the put-back's stop failed.
|
||||
func TestR1340_PutBackWithoutADatabase(t *testing.T) {
|
||||
msg, leg, reason := heldStoppedOperatorText(settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
|
||||
MixedCause: settings.MixedCausePutBack, FailedStep: "stopping the app before the put-back failed, so nothing was put back: compose down: timeout",
|
||||
ReplayError: "volume rm -f radicale_data: volume is in use", SafetyDump: "/mnt/hdd/felhom-data/backups/restore-undo/radicale/20261029T081800Z"})
|
||||
mustHave(t, msg, `"radicale"`, "off-site restore failed, and putting the live data back FAILED", "stopping the app before the put-back failed",
|
||||
"compose down: timeout", "kept at /mnt/hdd/felhom-data/backups/restore-undo/radicale/20261029T081800Z", "volume is in use")
|
||||
mustNotHave(t, msg, "database", "rolled back", optionCStory)
|
||||
if leg != "restore-hold-mixed" || reason != "put-back failed: stopping the app before the put-back failed, so nothing was put back" {
|
||||
t.Errorf("leg %q reason %q", leg, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1340_PutBackAfterADatabaseReplay(t *testing.T) {
|
||||
msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
|
||||
MixedCause: settings.MixedCausePutBack, DBReplay: true, FailedStep: "putting the live volumes back failed: tar xf failed"})
|
||||
mustHave(t, msg, "failed at the database replay", "putting the live volumes back failed: tar xf failed", "There is no copy of the live volumes")
|
||||
}
|
||||
|
||||
func TestR1340_DefinitionWrite(t *testing.T) {
|
||||
msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "gokapi", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
|
||||
MixedCause: settings.MixedCauseDefinitionWrite, FailedStep: "writing the live app definition back failed: app.yaml write failed", ReplayError: "app.yaml write failed"})
|
||||
mustHave(t, msg, "while writing the backup's app definition, before any data was touched", "writing the live app definition back failed", "data is the live data")
|
||||
mustNotHave(t, msg, "database", optionCStory)
|
||||
}
|
||||
|
||||
func TestR1340_LocalRestore(t *testing.T) {
|
||||
msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "app", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindLocal,
|
||||
MixedCause: settings.MixedCausePutBack, FailedStep: "putting the live volumes back failed: tar xf failed", SafetyDump: "/srv/undo/app/1"})
|
||||
mustHave(t, msg, "restore from its own backup failed", "putting the live volumes back failed")
|
||||
mustNotHave(t, msg, "off-site", "database", optionCStory)
|
||||
}
|
||||
|
||||
func TestR1340_PutBackPending(t *testing.T) {
|
||||
msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonPutBackPending, SafetyDump: "/mnt/hdd/undo/immich/1"})
|
||||
mustHave(t, msg, "copy (/mnt/hdd/undo/immich/1) is on a drive that is not bound", "poured back and the app started when the drive is bound", "DROPS that put-back")
|
||||
mustNotHave(t, msg, "database", optionCStory)
|
||||
if leg != "putback-pending" {
|
||||
t.Errorf("leg %q", leg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1340_InterruptedPutBack(t *testing.T) {
|
||||
msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonRestoreMixed, MixedCause: settings.MixedCauseInterruptedPutBack,
|
||||
FailedStep: "putting the interrupted restore's copy back failed: compose down: timeout", SafetyDump: "/mnt/hdd/undo/immich/1"})
|
||||
mustHave(t, msg, "cut by a stop could not be put back", "compose down: timeout", "kept at /mnt/hdd/undo/immich/1")
|
||||
mustNotHave(t, msg, "database", "off-site", optionCStory)
|
||||
}
|
||||
|
||||
// A mixed hold written before R-1340 names only what its record says.
|
||||
func TestR1340_LegacyMixedHoldAssumesNothing(t *testing.T) {
|
||||
msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "x", At: "2026-10-01T00:00:00Z", Reason: settings.HoldReasonRestoreMixed, ReplayError: "boom"})
|
||||
mustHave(t, msg, "stopped half-way", "The hold record says: boom")
|
||||
mustNotHave(t, msg, "database", optionCStory)
|
||||
}
|
||||
|
||||
// The notification reads the hold the writer persisted; when that write failed, it rebuilds one from the errors.
|
||||
func TestR1340_NotifyReadsTheHoldRecord(t *testing.T) {
|
||||
rec := settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed, MixedCause: settings.MixedCausePutBack, FailedStep: "x"}
|
||||
get := func(s string) (settings.RestoreHold, bool) { return rec, s == "radicale" }
|
||||
if h := heldHoldFromNotify(get, "radicale", errors.New("e"), nil); h.MixedCause != settings.MixedCausePutBack {
|
||||
t.Fatalf("the persisted record was not used: %+v", h)
|
||||
}
|
||||
h := heldHoldFromNotify(get, "other", errors.New("replay"), errors.New("rollback"))
|
||||
if h.Reason != settings.HoldReasonRestoreFailed || h.RollbackErr != "rollback" || h.ReplayError != "replay" {
|
||||
t.Fatalf("fallback hold %+v", h)
|
||||
}
|
||||
if h := heldHoldFromNotify(nil, "other", errors.New("replay"), nil); h.Reason != settings.HoldReasonRestoreMixed {
|
||||
t.Fatalf("no rollback error = mixed, got %+v", h)
|
||||
}
|
||||
}
|
||||
@@ -92,8 +92,9 @@ func main() {
|
||||
abandonStatus := flag.Bool("abandon-status", false, "R-241: print the state of this box's off-site abandonment countdown (set-aside path, due date, days left) and exit. Read-only.")
|
||||
abandonExtend := flag.Int("abandon-extend", 0, "R-241 (operator): extend a running abandonment countdown by N days from now, then exit. Refuses when no countdown is running.")
|
||||
abandonStop := flag.Bool("abandon-stop", false, "R-241 (operator): stop a running abandonment countdown, then exit. The set-aside history is kept and nothing is deleted. Refuses when no countdown is running.")
|
||||
listRestoreHolds := flag.Bool("restore-holds", false, "R-379 (operator): list apps this controller is deliberately holding stopped after a failed database restore whose rollback also failed, then exit. Read-only.")
|
||||
clearRestoreHold := flag.String("clear-restore-hold", "", "R-379 (operator): clear the restore hold on APP so it can start again, then exit. Clearing does NOT repair the database — check the app's data first; the undo copies are in its unit's db-dumps dir.")
|
||||
listRestoreHolds := flag.Bool("restore-holds", false, "R-379 (operator): list the apps this controller is holding stopped (every reason, with why), then exit. Read-only.")
|
||||
clearRestoreHold := flag.String("clear-restore-hold", "", "R-379/D64 (operator): clear the hold on APP (any reason), then exit. Run it inside the guest: "+holdClearCommandHint+
|
||||
". It asks the RUNNING controller, so no restart is needed; only when no controller runs does it edit settings.json. Clearing repairs nothing and starts nothing — check the app's data first.")
|
||||
flag.Parse()
|
||||
|
||||
if *showVersion {
|
||||
@@ -172,21 +173,23 @@ func main() {
|
||||
// starts, exactly like the other CLI subcommands: each loads config + settings, acts, and exits.
|
||||
// R-379 (operator): the way OUT of a hold. A CLI subcommand rather than an API route, matching
|
||||
// --abandon-stop above: the controller's HTTP surface is authenticated as the CUSTOMER, and
|
||||
// clearing a hold is a decision that should follow an operator looking at the database. Reaching
|
||||
// clearing a hold is a decision that should follow an operator looking at the data. Reaching
|
||||
// this needs `docker exec` into the guest, which is the operator gate this project already uses.
|
||||
// R-1343: the clear goes to the RUNNING controller over its local socket (holdclear.go); the list reads
|
||||
// settings.json, which the running controller saves at every hold change.
|
||||
if *listRestoreHolds || *clearRestoreHold != "" {
|
||||
cfg, err := config.LoadPermissive(*configPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: loading config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
lg := log.New(os.Stderr, "", 0)
|
||||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", lg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: loading settings: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *listRestoreHolds {
|
||||
lg := log.New(os.Stderr, "", 0)
|
||||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", lg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: loading settings: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
holds := sett.ListRestoreHolds()
|
||||
if len(holds) == 0 {
|
||||
fmt.Println("no restore holds in force")
|
||||
@@ -197,34 +200,9 @@ func main() {
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
cleared, err := sett.ClearRestoreHold(*clearRestoreHold)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: clearing %s: %v\n", *clearRestoreHold, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if !cleared {
|
||||
// "there was nothing to clear" is not success — a silent 0 here would let an operator
|
||||
// believe they had unblocked an app whose name they mistyped.
|
||||
fmt.Fprintf(os.Stderr, "no restore hold in force for %q — nothing was changed\n", *clearRestoreHold)
|
||||
os.Exit(2)
|
||||
}
|
||||
// THE RESTART IS NOT OPTIONAL, and saying so is the difference between a route that works
|
||||
// and one an operator believes worked. This runs as a SECOND process: it clears the hold in
|
||||
// settings.json, but the RUNNING controller holds its own in-memory Settings and keeps
|
||||
// refusing until it reloads. Measured on demo-hp 2026-08-22 — the clear succeeded, the file
|
||||
// was correct, and the customer's start button still refused until the controller restarted.
|
||||
//
|
||||
// There is also a lost-update window while both processes hold the file: if the running
|
||||
// controller saves settings.json from its own copy before the restart, the clear is undone.
|
||||
// Restarting promptly closes it. A route that clears through the running controller would
|
||||
// remove both problems and is the right shape later; it needs an operator tier the
|
||||
// controller's HTTP surface does not have today (it authenticates as the CUSTOMER, and a
|
||||
// customer clearing their own hold is precisely what the hold exists to prevent).
|
||||
fmt.Printf("restore hold cleared for %s in settings.json.\n", *clearRestoreHold)
|
||||
fmt.Printf(" NOW RESTART THE CONTROLLER, or it will keep refusing to start the app:\n")
|
||||
fmt.Printf(" systemctl restart felhom-controller-bootstrap.service\n")
|
||||
fmt.Printf(" Check the app's data first — the undo copies are in its unit's db-dumps dir.\n")
|
||||
os.Exit(0)
|
||||
// R-1343: through the RUNNING controller (holdclear.go) — the old second-process edit of settings.json was
|
||||
// written back by the running controller's next save. Only when no controller answers is the file edited here.
|
||||
os.Exit(clearRestoreHoldCLI(cfg.Paths.DataDir, *clearRestoreHold, os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
if *abandonStatus || *abandonExtend > 0 || *abandonStop {
|
||||
@@ -485,6 +463,18 @@ func main() {
|
||||
return stackMgr.DriveLive(hdd)
|
||||
})
|
||||
appStopRecovery := appStopGuard.Recover()
|
||||
// D64 = B / R-1343: the operator's hold clear — ONE door for the hub's clear_hold and the box command, run in THIS
|
||||
// process so memory and settings.json agree (holdclear.go). The box command reaches it over a unix socket in the
|
||||
// data dir (0600, nothing on TCP). A socket that cannot be bound is logged; the command then says plainly that no
|
||||
// running controller answered.
|
||||
holdDeps := holdClearDeps{clear: appStopGuard.ClearHoldByOperator, forgetUpdateHeld: stackMgr.ForgetUpdateHeld, logger: logger}
|
||||
if holdLn, herr := listenHoldCtl(holdCtlPath(cfg.Paths.DataDir), func(stack, who string) (string, error) {
|
||||
return clearHoldNow(holdDeps, stack, who)
|
||||
}, logger); herr != nil {
|
||||
logger.Printf("[ERROR] [hold] the local hold-clear channel could not be opened: %v — `--clear-restore-hold` will edit settings.json and need a restart", herr)
|
||||
} else {
|
||||
defer holdLn.Close()
|
||||
}
|
||||
// R-1251: a put-back that waited for its drive (a hold from an earlier start) runs now if that drive is bound.
|
||||
if started := appStopGuard.ResumeWaitingPutBacks(); len(started) > 0 {
|
||||
logger.Printf("[WARN] [appstop] %d app(s) whose interrupted restore waited for its drive are put back and started: %v", len(started), started)
|
||||
@@ -1081,6 +1071,7 @@ func main() {
|
||||
holdReporter.Tick(time.Now())
|
||||
return nil
|
||||
})
|
||||
oomLog := &oomLogGate{} // R-1341
|
||||
sched.Every("deadapp-check", 30*time.Second, func(ctx context.Context) error {
|
||||
if bootGrace.Within(time.Now()) {
|
||||
return nil // still inside the startup settle window (90 s, or ~15 min after a crash boot)
|
||||
@@ -1096,8 +1087,12 @@ func main() {
|
||||
if oerr != nil {
|
||||
logger.Printf("[WARN] [deadapp] OOM scan failed: %v", oerr)
|
||||
} else {
|
||||
for _, o := range ooms {
|
||||
// R-1341: the line once per NEW kill (oomlog.go); the event call stays per scan — the notifier's cooldown
|
||||
// keeps it to one, as before.
|
||||
for _, o := range oomLog.fresh(ooms) {
|
||||
logger.Printf("[WARN] [deadapp] %s: container %s was OOM-killed (started %s; kills %d, limit %s, peak %s; seen by %s)", o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
|
||||
}
|
||||
for _, o := range ooms {
|
||||
notifier.NotifyAppOOMFrom(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
|
||||
}
|
||||
}
|
||||
@@ -1217,7 +1212,7 @@ func main() {
|
||||
// R-314/R-279/R-177 (`09` §3 decision 185): the operator's closed list of actions, carried in the
|
||||
// report reply and run in THIS process (opactions_wiring.go). The result rides the next report;
|
||||
// a finished result fires an out-of-cycle report through the trigger once it exists.
|
||||
opActions := report.NewOperatorActions(ctx, operatorActionHandlers(backupMgr, sched, func() {
|
||||
opActions := report.NewOperatorActions(ctx, operatorActionHandlers(backupMgr, sched, holdDeps, func() {
|
||||
if f, ok := opActionReportFire.Load().(func()); ok && f != nil {
|
||||
f()
|
||||
}
|
||||
@@ -1548,22 +1543,18 @@ func main() {
|
||||
// DELIBERATELY NOT `backup_failed`: customer-enabled by default, Hungarian "A biztonsági
|
||||
// mentés sikertelen!", and this is a deliberate hold rather than a backup failure. R-171 one
|
||||
// path over.
|
||||
// R-1340: the text is built from the HOLD RECORD the writer persisted just before this call (holdmail.go) — its
|
||||
// reason, which restore, which branch, which put-back step failed, whether it was a database replay — never the
|
||||
// option-C database story for every cause.
|
||||
backupMgr.SetRestoreHoldNotify(func(stack string, replayErr, rollbackErr error) {
|
||||
d := notify.BackupRunFailuresDetails{RunKind: "offsite-reconstitute", Failed: 1, Attempted: 1}
|
||||
if rollbackErr == nil { // R-893: the rollback WORKED, but files/volumes/version had already moved
|
||||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: stack, Leg: "restore-hold-mixed", Reason: "replay failed after files/volumes/version moved"})
|
||||
msg := fmt.Sprintf("App %q is HELD STOPPED for support: its off-site database restore failed. The database was rolled back to the pre-restore copy and the live definition written back, "+
|
||||
"but its files and named volumes are the snapshot's — a mixed state (R-893). The app will not start from any path until the hold is cleared. Replay error: %v", stack, replayErr)
|
||||
notifier.NotifyBackupRunFailures(msg, d)
|
||||
return
|
||||
h := heldHoldFromNotify(sett.GetRestoreHold, stack, replayErr, rollbackErr)
|
||||
msg, leg, reason := heldStoppedOperatorText(h)
|
||||
runKind := "offsite-reconstitute"
|
||||
if h.RestoreKind == settings.RestoreKindLocal {
|
||||
runKind = "restore"
|
||||
}
|
||||
reason := "rollback failed"
|
||||
if rollbackErr != nil {
|
||||
reason = rollbackErr.Error()
|
||||
}
|
||||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: stack, Leg: "restore-hold", Reason: reason})
|
||||
msg := fmt.Sprintf("App %q is HELD STOPPED: its off-site database restore failed AND the rollback to the customer's own pre-restore copy also failed. "+
|
||||
"The app will not start from any path until the hold is cleared. Replay error: %v. Rollback error: %v", stack, replayErr, rollbackErr)
|
||||
d := notify.BackupRunFailuresDetails{RunKind: runKind, Failed: 1, Attempted: 1}
|
||||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: stack, Leg: leg, Reason: reason})
|
||||
notifier.NotifyBackupRunFailures(msg, d)
|
||||
})
|
||||
// R-893 option A (D20 = A): a failed off-site restore that was PUT BACK — the live volumes and definition poured
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// ── R-1341: the OOM line is logged once per NEW kill ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// ScanOOMKilled reports a container for as long as it lives after a kill (Docker's OOMKilled flag is sticky, and the
|
||||
// kernel's kill counter never goes down), so the 30-s dead-app scan logged „[WARN] [deadapp] … was OOM-killed" about
|
||||
// 2 880 times a day per container — measured 2026-10-29 on Tester 1 — while the notifier's cooldown kept the event to
|
||||
// one. The line now goes out when the kill is NEW: the key is (container, its start time, the kill count), and a line
|
||||
// is logged only when a container's key changes — a second scan of the same kill logs nothing, a new kill (the count
|
||||
// went up) or a restarted container (a new start time) logs again. The event call is unchanged (every scan, the
|
||||
// notifier decides). Pinned by TestR1341_*.
|
||||
|
||||
// oomLogGate remembers the last logged key per container. Not safe for concurrent use: the dead-app job runs serially.
|
||||
type oomLogGate struct {
|
||||
last map[string]string
|
||||
}
|
||||
|
||||
func oomLogKey(o stacks.OOMContainer) string {
|
||||
return o.StartedAt + "|" + strconv.FormatInt(o.Kills, 10)
|
||||
}
|
||||
|
||||
// fresh returns the containers whose kill is new since the last scan, and remembers this scan's keys. A container
|
||||
// absent from this scan is forgotten (it was removed or restarted clean), which also bounds the memory.
|
||||
func (g *oomLogGate) fresh(ooms []stacks.OOMContainer) []stacks.OOMContainer {
|
||||
next := make(map[string]string, len(ooms))
|
||||
var out []stacks.OOMContainer
|
||||
for _, o := range ooms {
|
||||
k := oomLogKey(o)
|
||||
next[o.Container] = k
|
||||
if prev, ok := g.last[o.Container]; ok && prev == k {
|
||||
continue
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
g.last = next
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// R-1341: the „[WARN] [deadapp] … was OOM-killed" line goes out once per NEW kill, not every 30-s scan.
|
||||
//
|
||||
// RED-PROOF (2026-10-29): with oomLogGate.fresh returning its input unchanged (the pre-R-1341 loop logged every
|
||||
// container every scan), TestR1341_ASecondScanOfTheSameKillLogsNothing fails on „logged 1 line(s), want 0".
|
||||
|
||||
func oom(container, started string, kills int64) stacks.OOMContainer {
|
||||
return stacks.OOMContainer{Stack: "gokapi", Container: container, StartedAt: started, Kills: kills}
|
||||
}
|
||||
|
||||
func TestR1341_ASecondScanOfTheSameKillLogsNothing(t *testing.T) {
|
||||
g := &oomLogGate{}
|
||||
scan := []stacks.OOMContainer{oom("gokapi", "2026-10-29T09:02:00Z", 1)}
|
||||
if got := g.fresh(scan); len(got) != 1 {
|
||||
t.Fatalf("the first sighting must log, logged %d line(s)", len(got))
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
if got := g.fresh(scan); len(got) != 0 {
|
||||
t.Fatalf("scan %d of the same kill logged %d line(s), want 0", i+2, len(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1341_ANewKillLogsAgain(t *testing.T) {
|
||||
g := &oomLogGate{}
|
||||
g.fresh([]stacks.OOMContainer{oom("gokapi", "2026-10-29T09:02:00Z", 1)})
|
||||
got := g.fresh([]stacks.OOMContainer{oom("gokapi", "2026-10-29T09:02:00Z", 2)})
|
||||
if len(got) != 1 || got[0].Kills != 2 {
|
||||
t.Fatalf("a new kill (count 1→2) must log once, got %+v", got)
|
||||
}
|
||||
if got := g.fresh([]stacks.OOMContainer{oom("gokapi", "2026-10-29T09:02:00Z", 2)}); len(got) != 0 {
|
||||
t.Fatalf("the same new kill must not log twice, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1341_ARestartedContainerLogsAgain(t *testing.T) {
|
||||
g := &oomLogGate{}
|
||||
g.fresh([]stacks.OOMContainer{oom("gokapi", "2026-10-29T09:02:00Z", 1)})
|
||||
if got := g.fresh([]stacks.OOMContainer{oom("gokapi", "2026-10-29T09:30:00Z", 1)}); len(got) != 1 {
|
||||
t.Fatalf("a restarted container's kill must log, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Two containers are judged apart; one that leaves the scan is forgotten, so its return logs again.
|
||||
func TestR1341_ContainersAreJudgedApart(t *testing.T) {
|
||||
g := &oomLogGate{}
|
||||
a, b := oom("a", "t1", 1), oom("b", "t1", 1)
|
||||
if got := g.fresh([]stacks.OOMContainer{a, b}); len(got) != 2 {
|
||||
t.Fatalf("first scan: %+v", got)
|
||||
}
|
||||
if got := g.fresh([]stacks.OOMContainer{a}); len(got) != 0 {
|
||||
t.Fatalf("a seen again must not log: %+v", got)
|
||||
}
|
||||
if got := g.fresh([]stacks.OOMContainer{a, b}); len(got) != 1 || got[0].Container != "b" {
|
||||
t.Fatalf("b came back after leaving the scan: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -17,18 +17,26 @@ import (
|
||||
// executor is built earlier than the trigger, so it reads it through this holder.
|
||||
var opActionReportFire atomic.Value
|
||||
|
||||
// operatorActionHandlers wires the four operator actions (R-314/R-279/R-177, `09` §3 decision 185)
|
||||
// operatorActionHandlers wires the five operator actions (R-314/R-279/R-177, `09` §3 decision 185)
|
||||
// to the running process. ONE wiring site, called from main and from the wiring test, so the test
|
||||
// exercises the doors production uses (the seam-built-but-never-wired trap).
|
||||
//
|
||||
// offsite_backup_now makes the SAME four checks as the household's „back up now"
|
||||
// (offboxRunHandler) and then runs the same call, so an operator press can never do what the
|
||||
// household's own press cannot.
|
||||
func operatorActionHandlers(backupMgr *backup.Manager, sched *scheduler.Scheduler, resultReady func(), logger *log.Logger) report.OperatorActionHandlers {
|
||||
//
|
||||
// clear_hold (D64 = B) goes through clearHoldNow with hold — the same door the box command's socket uses
|
||||
// (holdclear.go), so the hub's button and the box command can never differ. hold.clear nil = not available.
|
||||
func operatorActionHandlers(backupMgr *backup.Manager, sched *scheduler.Scheduler, hold holdClearDeps, resultReady func(), logger *log.Logger) report.OperatorActionHandlers {
|
||||
h := report.OperatorActionHandlers{ResultReady: resultReady}
|
||||
if sched != nil {
|
||||
h.RunJob = sched.RunNow
|
||||
}
|
||||
if hold.clear != nil {
|
||||
h.ClearHold = func(stack string, actionID int64) (string, error) {
|
||||
return clearHoldNow(hold, stack, fmt.Sprintf("the operator via the hub, action #%d", actionID))
|
||||
}
|
||||
}
|
||||
if backupMgr == nil {
|
||||
return h
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ func TestD1_AbandonStopActsOnTheRunningManagerAndDisk(t *testing.T) {
|
||||
t.Fatal("fixture: a countdown must be running")
|
||||
}
|
||||
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, nil, logger), logger)
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, holdClearDeps{}, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpAbandonStop}})
|
||||
|
||||
if mgr.AbandonStatus().Active {
|
||||
@@ -97,7 +97,7 @@ func TestD1_OffsiteBackupNowRefusesWithoutATarget(t *testing.T) {
|
||||
t.Fatal("no restic may run")
|
||||
return nil, nil
|
||||
})
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, nil, logger), logger)
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, holdClearDeps{}, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 9, Action: report.OpOffsiteBackupNow}})
|
||||
if r := o.Results(); len(r) != 1 || r[0].Outcome != report.OutcomeRefused {
|
||||
t.Fatalf("result = %+v", r)
|
||||
@@ -115,7 +115,7 @@ func TestD1_RunJobUsesTheScheduler(t *testing.T) {
|
||||
defer cancel()
|
||||
sched.Start(ctx)
|
||||
defer sched.Stop()
|
||||
o := report.NewOperatorActions(ctx, operatorActionHandlers(nil, sched, nil, logger), logger)
|
||||
o := report.NewOperatorActions(ctx, operatorActionHandlers(nil, sched, holdClearDeps{}, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpRunJob, Arg: "fill-watch"}, {ID: 2, Action: report.OpRunJob, Arg: "offsite-proof"}})
|
||||
select {
|
||||
case od := <-ran:
|
||||
|
||||
Reference in new issue
Block a user