D65 = A: the busy file the agent reads before an operator reboot, and the household's 'Felhom support restarted your box' line; R-1350: a hold clear records the wanted stop so no app_start_failed follows it
gates / gates (push) Successful in 32s

This commit is contained in:
admin committed 2026-10-29 16:24:40 +01:00
1 parent a28db2d17f
commit b69e90bdad
22 files changed
+662 -11

No files matched your search

@@ -0,0 +1,49 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/busymark"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)
// D65 = A: the agent reads ONE fixed guest path — the controller-data volume + data/ctl/busy
// (felhom-agent ControllerBusyMarkerGuestPath, and its sudoers line). The volume is mounted at
// /opt/docker/felhom-controller, so the controller must write <default data dir>/ctl/busy.
// RED-PROOF: rename holdCtlDir or busymark.FileName → the path moves → FAILS.
func TestBusyMarkPathIsTheOneTheAgentReads(t *testing.T) {
cfg, err := config.LoadFromBytes([]byte("customer:\n id: t\n domain: t.example\n"))
if err != nil {
t.Fatal(err)
}
got := filepath.Join(cfg.Paths.DataDir, holdCtlDir, busymark.FileName)
want := "/opt/docker/felhom-controller/data/ctl/busy" // = the agent's /var/lib/docker/volumes/felhom-controller-data/_data/data/ctl/busy
if got != want {
t.Fatalf("the busy file is %s, the agent reads %s — an operator reboot would never be refused for a backup", got, want)
}
}
// Every kind of work has its probe and the writer is started — a seam built and never wired is this project's commonest
// defect. RED-PROOF: delete any AddProbe line or the Run → FAILS.
func TestBusyMarkIsWiredAtStartup(t *testing.T) {
raw, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
src := string(raw)
for _, want := range []string{
`busyMark.AddProbe("an app update is running", stackMgr.AnyUpdating)`,
`return backupMgr.IsRunning() || backupMgr.RestoreStatus().Running`,
`busyMark.AddProbe("the night backup catch-up is running", catchUp.Running)`,
`busyMark.AddProbe("an app export or import is running", appExporter.IsRunning)`,
`go busyMark.Run(ctx)`,
`go webServer.WatchOperatorReboot(ctx)`, // the household line after an operator reboot
} {
if !strings.Contains(src, want) {
t.Fatalf("main.go does not contain %q", want)
}
}
}
+19 -4
View File
@@ -31,9 +31,12 @@ import (
// holdClearDeps are clearHoldNow's seams (production: holdClearDepsFor).
type holdClearDeps struct {
clear func(stack string) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperator
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
logger *log.Logger
clear func(stack string, beforeLift func()) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperatorThen
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
// keepStopped (R-1350) records that the stop is wanted for an app that is not up — run once the hold is found and
// BEFORE it is lifted: stacks.Manager.RecordStopWantedIfDown (nil = none).
keepStopped func(stack string) (bool, error)
logger *log.Logger
}
// clearHoldNow lifts stack's hold and returns the result sentence (English, operator-facing). who names the person
@@ -43,7 +46,19 @@ func clearHoldNow(d holdClearDeps, stack, who string) (string, error) {
if d.clear == nil {
return "", fmt.Errorf("%w: clearing a hold is not available on this box", report.ErrActionRefused)
}
h, err := d.clear(stack)
var before func()
if d.keepStopped != nil {
before = func() {
rec, kerr := d.keepStopped(stack)
switch {
case kerr != nil && d.logger != nil:
d.logger.Printf("[WARN] [hold] %s: the stop could not be recorded as wanted (%v) — the dead-app alarm may follow the clear (R-1350)", stack, kerr)
case rec && d.logger != nil:
d.logger.Printf("[INFO] [hold] %s: the stop is recorded as wanted (desired state: stopped) — no dead-app alarm; the household's Start ends it (R-1350)", stack)
}
}
}
h, err := d.clear(stack, before)
switch {
case errors.Is(err, backup.ErrHoldBadStack):
return "", fmt.Errorf("%w: %q is not a valid app name — nothing was changed", report.ErrActionRefused, stack)
+1 -1
View File
@@ -76,7 +76,7 @@ func newD64Rig(t *testing.T) *d64Rig {
r.guard = backup.NewAppStopGuard(filepath.Join(r.dir, "appstop-state.json"), r.logger)
r.guard.SetHolds(sett)
r.holdDeps = holdClearDeps{
clear: r.guard.ClearHoldByOperator,
clear: r.guard.ClearHoldByOperatorThen,
forgetUpdateHeld: func(s string) bool { r.forgot = append(r.forgot, s); return true },
logger: r.logger,
}
+15 -1
View File
@@ -34,6 +34,7 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
"gitea.dooplex.hu/admin/felhom-controller/internal/busymark"
"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
@@ -467,7 +468,8 @@ func main() {
// process so memory and settings.json agree (holdclear.go). The box command reaches it over a unix socket in the
// data dir (0600, nothing on TCP). A socket that cannot be bound is logged; the command then says plainly that no
// running controller answered.
holdDeps := holdClearDeps{clear: appStopGuard.ClearHoldByOperator, forgetUpdateHeld: stackMgr.ForgetUpdateHeld, logger: logger}
holdDeps := holdClearDeps{clear: appStopGuard.ClearHoldByOperatorThen, forgetUpdateHeld: stackMgr.ForgetUpdateHeld,
keepStopped: stackMgr.RecordStopWantedIfDown, logger: logger}
if holdLn, herr := listenHoldCtl(holdCtlPath(cfg.Paths.DataDir), func(stack, who string) (string, error) {
return clearHoldNow(holdDeps, stack, who)
}, logger); herr != nil {
@@ -577,6 +579,11 @@ func main() {
logger.Println("[INFO] Healthchecks ping UUIDs configured but no longer used — monitoring is now handled by the Hub")
}
// D65 = A (2026-10-29): the busy file the host agent reads before an operator reboot of the guest (internal/busymark).
// Probes are added where each component is built; the writer starts once the last one exists (after the exporter).
busyMark := &busymark.Writer{Path: filepath.Join(cfg.Paths.DataDir, holdCtlDir, busymark.FileName), Logger: logger}
busyMark.AddProbe("an app update is running", stackMgr.AnyUpdating)
// --- Initialize backup manager (app-data only: DB dumps + Docker-volume tars) ---
var backupMgr *backup.Manager
// R-550: the restore the last stop interrupted, found when the record loads; raised once the notifier exists.
@@ -588,6 +595,9 @@ func main() {
}
if cfg.Backup.Enabled {
backupMgr = backup.NewManager(cfg, sett, logger)
busyMark.AddProbe("a backup, a restore or an off-site run is running", func() bool {
return backupMgr.IsRunning() || backupMgr.RestoreStatus().Running
})
// The stray-copy fix: a second-drive copy container a previous controller process lost (restart, self-update)
// still runs `rsync --delete`. Remove every one BEFORE the scheduler or the web server can start a backup or a
// restore; until that succeeds, restores are refused. Pinned by TestStrayCopy_ClearIsWiredBeforeTheScheduler.
@@ -1661,6 +1671,7 @@ func main() {
if quiesceLoop != nil {
quiesceLoop.SetCatchUpFn(catchUp.Running)
}
busyMark.AddProbe("the night backup catch-up is running", catchUp.Running)
catchUp.Evaluate(ctx, "controller start")
resumeTick := time.NewTicker(time.Minute)
go nightchain.ResumeWatch(ctx, resumeTick.C,
@@ -2119,6 +2130,8 @@ func main() {
// this the export path would be the uncovered sibling of two covered ones, which is how a reader
// concludes the whole class is handled (§2.2).
appExporter.SetStopGuard(exportStopGuard{g: appStopGuard})
busyMark.AddProbe("an app export or import is running", appExporter.IsRunning)
go busyMark.Run(ctx)
// R-1279 (D48 for the import): the import downloads first, with the restores' helper and sentences.
if backupMgr != nil {
appExporter.SetImagePreparer(backupMgr.PrepareRestoreImages)
@@ -2127,6 +2140,7 @@ func main() {
// --- Initialize web server ---
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, sched, sett, alertMgr, notifier, updater, logger, Version)
go webServer.WatchOperatorReboot(ctx) // D65: „Felhom support restarted your box at …" after an operator reboot
// v0.316.0: which BUILD this is, not just which version it calls itself. Every shell's stylesheet
// URL is derived from it, so two builds of one version can never share a cache entry. Init-only,
// before the HTTP server starts.
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"os"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// R-1350 (measured 2026-10-29 14:57 on Tester 1): the operator cleared gokapi's crash-loop hold from the hub, the app
// stayed stopped (the clear starts nothing), and 30 s later the dead-app scan sent „Telepített alkalmazás nem fut:
// Gokapi" — the hold had kept the app out of the scan, and the household's recorded wish was still „running".
// The fix: the clear records that the stop is WANTED (desired state „stopped") for an app that is not up, BEFORE the
// hold is lifted; a later Start records „running" and ends it.
// The consequence, through the scan's own derivation: a stopped app whose wish is „running" is DOWN (the alarm); the
// same app with the wish „stopped" is not.
func TestR1350_AStopWantedIsNotDown(t *testing.T) {
st := func(desired string) []stacks.Stack {
return []stacks.Stack{{Name: "gokapi", Deployed: true, State: stacks.StateStopped, AppConfig: &stacks.AppConfig{DesiredState: desired}}}
}
if _, states := classifyRunStates(st(stacks.DesiredStateRunning), nil, nil, time.Now()); len(states) != 1 || !states[0].Down {
t.Fatalf("fixture: a stopped app wished running must be down: %+v", states)
}
if _, states := classifyRunStates(st(stacks.DesiredStateStopped), nil, nil, time.Now()); len(states) != 1 || states[0].Down {
t.Fatalf("a stopped app wished stopped is down — the alarm fires: %+v", states)
}
}
// The clear records the wish BEFORE the hold is lifted (a dead-app scan between the two would read „no hold, wished
// running" and mail), and only for an app that IS held. RED-PROOF: drop the keepStopped call from the clear → FAILS
// („not recorded"); call it after the lift → FAILS („after the hold was lifted").
func TestR1350_TheClearRecordsTheStopIsWanted(t *testing.T) {
r := newD64Rig(t)
r.hold(t, settings.RestoreHold{Stack: "gokapi", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop", Trip: 1})
var recorded []string
var heldWhenRecorded bool
r.holdDeps.keepStopped = func(stack string) (bool, error) {
recorded = append(recorded, stack)
_, heldWhenRecorded = r.sett.GetRestoreHold(stack)
return true, nil
}
o := r.actions()
o.Reconcile([]report.OperatorAction{{ID: 21, Action: report.OpClearHold, Arg: "gokapi"}})
if res := onlyResult(t, o, 21); res.Outcome != report.OutcomeDone {
t.Fatalf("result %+v", res)
}
if len(recorded) != 1 || recorded[0] != "gokapi" {
t.Fatalf("the stop was not recorded as wanted: %v", recorded)
}
if !heldWhenRecorded {
t.Fatal("the wish was recorded after the hold was lifted — a scan in between mails app_start_failed")
}
if lg := r.logBuf.String(); !strings.Contains(lg, "gokapi: the stop is recorded as wanted") {
t.Fatalf("the box's log must say it:\n%s", lg)
}
// a refusal (no hold) records nothing — a down app with no hold is a real fault the alarm must keep
o.Reconcile([]report.OperatorAction{{ID: 22, Action: report.OpClearHold, Arg: "kimai"}})
if res := onlyResult(t, o, 22); res.Outcome != report.OutcomeRefused {
t.Fatalf("result %+v", res)
}
if len(recorded) != 1 {
t.Fatalf("a refused clear recorded a wish: %v", recorded)
}
}
// The running controller's clear is wired with the record. RED-PROOF: drop keepStopped from main.go's holdDeps → FAILS.
func TestR1350_MainWiresTheRecord(t *testing.T) {
raw, err := os.ReadFile("main.go")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), "keepStopped: stackMgr.RecordStopWantedIfDown") || !strings.Contains(string(raw), "clear: appStopGuard.ClearHoldByOperatorThen") {
t.Fatal("main.go's hold clear does not record the wanted stop (R-1350)")
}
}