D65 = A: the busy file the agent reads before an operator reboot, and the household's 'Felhom support restarted your box' line; R-1350: a hold clear records the wanted stop so no app_start_failed follows it
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
This commit is contained in:
1 parent
a28db2d17f
commit
b69e90bdad
22 files changed
+662
-11
No files matched your search
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/busymark"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
)
|
||||
|
||||
// D65 = A: the agent reads ONE fixed guest path — the controller-data volume + data/ctl/busy
|
||||
// (felhom-agent ControllerBusyMarkerGuestPath, and its sudoers line). The volume is mounted at
|
||||
// /opt/docker/felhom-controller, so the controller must write <default data dir>/ctl/busy.
|
||||
// RED-PROOF: rename holdCtlDir or busymark.FileName → the path moves → FAILS.
|
||||
func TestBusyMarkPathIsTheOneTheAgentReads(t *testing.T) {
|
||||
cfg, err := config.LoadFromBytes([]byte("customer:\n id: t\n domain: t.example\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := filepath.Join(cfg.Paths.DataDir, holdCtlDir, busymark.FileName)
|
||||
want := "/opt/docker/felhom-controller/data/ctl/busy" // = the agent's /var/lib/docker/volumes/felhom-controller-data/_data/data/ctl/busy
|
||||
if got != want {
|
||||
t.Fatalf("the busy file is %s, the agent reads %s — an operator reboot would never be refused for a backup", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every kind of work has its probe and the writer is started — a seam built and never wired is this project's commonest
|
||||
// defect. RED-PROOF: delete any AddProbe line or the Run → FAILS.
|
||||
func TestBusyMarkIsWiredAtStartup(t *testing.T) {
|
||||
raw, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := string(raw)
|
||||
for _, want := range []string{
|
||||
`busyMark.AddProbe("an app update is running", stackMgr.AnyUpdating)`,
|
||||
`return backupMgr.IsRunning() || backupMgr.RestoreStatus().Running`,
|
||||
`busyMark.AddProbe("the night backup catch-up is running", catchUp.Running)`,
|
||||
`busyMark.AddProbe("an app export or import is running", appExporter.IsRunning)`,
|
||||
`go busyMark.Run(ctx)`,
|
||||
`go webServer.WatchOperatorReboot(ctx)`, // the household line after an operator reboot
|
||||
} {
|
||||
if !strings.Contains(src, want) {
|
||||
t.Fatalf("main.go does not contain %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,9 +31,12 @@ import (
|
||||
|
||||
// holdClearDeps are clearHoldNow's seams (production: holdClearDepsFor).
|
||||
type holdClearDeps struct {
|
||||
clear func(stack string) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperator
|
||||
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
|
||||
logger *log.Logger
|
||||
clear func(stack string, beforeLift func()) (settings.RestoreHold, error) // backup.AppStopGuard.ClearHoldByOperatorThen
|
||||
forgetUpdateHeld func(stack string) bool // stacks.Manager.ForgetUpdateHeld (nil = none)
|
||||
// keepStopped (R-1350) records that the stop is wanted for an app that is not up — run once the hold is found and
|
||||
// BEFORE it is lifted: stacks.Manager.RecordStopWantedIfDown (nil = none).
|
||||
keepStopped func(stack string) (bool, error)
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// clearHoldNow lifts stack's hold and returns the result sentence (English, operator-facing). who names the person
|
||||
@@ -43,7 +46,19 @@ func clearHoldNow(d holdClearDeps, stack, who string) (string, error) {
|
||||
if d.clear == nil {
|
||||
return "", fmt.Errorf("%w: clearing a hold is not available on this box", report.ErrActionRefused)
|
||||
}
|
||||
h, err := d.clear(stack)
|
||||
var before func()
|
||||
if d.keepStopped != nil {
|
||||
before = func() {
|
||||
rec, kerr := d.keepStopped(stack)
|
||||
switch {
|
||||
case kerr != nil && d.logger != nil:
|
||||
d.logger.Printf("[WARN] [hold] %s: the stop could not be recorded as wanted (%v) — the dead-app alarm may follow the clear (R-1350)", stack, kerr)
|
||||
case rec && d.logger != nil:
|
||||
d.logger.Printf("[INFO] [hold] %s: the stop is recorded as wanted (desired state: stopped) — no dead-app alarm; the household's Start ends it (R-1350)", stack)
|
||||
}
|
||||
}
|
||||
}
|
||||
h, err := d.clear(stack, before)
|
||||
switch {
|
||||
case errors.Is(err, backup.ErrHoldBadStack):
|
||||
return "", fmt.Errorf("%w: %q is not a valid app name — nothing was changed", report.ErrActionRefused, stack)
|
||||
|
||||
@@ -76,7 +76,7 @@ func newD64Rig(t *testing.T) *d64Rig {
|
||||
r.guard = backup.NewAppStopGuard(filepath.Join(r.dir, "appstop-state.json"), r.logger)
|
||||
r.guard.SetHolds(sett)
|
||||
r.holdDeps = holdClearDeps{
|
||||
clear: r.guard.ClearHoldByOperator,
|
||||
clear: r.guard.ClearHoldByOperatorThen,
|
||||
forgetUpdateHeld: func(s string) bool { r.forgot = append(r.forgot, s); return true },
|
||||
logger: r.logger,
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/busymark"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth"
|
||||
cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
@@ -467,7 +468,8 @@ func main() {
|
||||
// process so memory and settings.json agree (holdclear.go). The box command reaches it over a unix socket in the
|
||||
// data dir (0600, nothing on TCP). A socket that cannot be bound is logged; the command then says plainly that no
|
||||
// running controller answered.
|
||||
holdDeps := holdClearDeps{clear: appStopGuard.ClearHoldByOperator, forgetUpdateHeld: stackMgr.ForgetUpdateHeld, logger: logger}
|
||||
holdDeps := holdClearDeps{clear: appStopGuard.ClearHoldByOperatorThen, forgetUpdateHeld: stackMgr.ForgetUpdateHeld,
|
||||
keepStopped: stackMgr.RecordStopWantedIfDown, logger: logger}
|
||||
if holdLn, herr := listenHoldCtl(holdCtlPath(cfg.Paths.DataDir), func(stack, who string) (string, error) {
|
||||
return clearHoldNow(holdDeps, stack, who)
|
||||
}, logger); herr != nil {
|
||||
@@ -577,6 +579,11 @@ func main() {
|
||||
logger.Println("[INFO] Healthchecks ping UUIDs configured but no longer used — monitoring is now handled by the Hub")
|
||||
}
|
||||
|
||||
// D65 = A (2026-10-29): the busy file the host agent reads before an operator reboot of the guest (internal/busymark).
|
||||
// Probes are added where each component is built; the writer starts once the last one exists (after the exporter).
|
||||
busyMark := &busymark.Writer{Path: filepath.Join(cfg.Paths.DataDir, holdCtlDir, busymark.FileName), Logger: logger}
|
||||
busyMark.AddProbe("an app update is running", stackMgr.AnyUpdating)
|
||||
|
||||
// --- Initialize backup manager (app-data only: DB dumps + Docker-volume tars) ---
|
||||
var backupMgr *backup.Manager
|
||||
// R-550: the restore the last stop interrupted, found when the record loads; raised once the notifier exists.
|
||||
@@ -588,6 +595,9 @@ func main() {
|
||||
}
|
||||
if cfg.Backup.Enabled {
|
||||
backupMgr = backup.NewManager(cfg, sett, logger)
|
||||
busyMark.AddProbe("a backup, a restore or an off-site run is running", func() bool {
|
||||
return backupMgr.IsRunning() || backupMgr.RestoreStatus().Running
|
||||
})
|
||||
// The stray-copy fix: a second-drive copy container a previous controller process lost (restart, self-update)
|
||||
// still runs `rsync --delete`. Remove every one BEFORE the scheduler or the web server can start a backup or a
|
||||
// restore; until that succeeds, restores are refused. Pinned by TestStrayCopy_ClearIsWiredBeforeTheScheduler.
|
||||
@@ -1661,6 +1671,7 @@ func main() {
|
||||
if quiesceLoop != nil {
|
||||
quiesceLoop.SetCatchUpFn(catchUp.Running)
|
||||
}
|
||||
busyMark.AddProbe("the night backup catch-up is running", catchUp.Running)
|
||||
catchUp.Evaluate(ctx, "controller start")
|
||||
resumeTick := time.NewTicker(time.Minute)
|
||||
go nightchain.ResumeWatch(ctx, resumeTick.C,
|
||||
@@ -2119,6 +2130,8 @@ func main() {
|
||||
// this the export path would be the uncovered sibling of two covered ones, which is how a reader
|
||||
// concludes the whole class is handled (§2.2).
|
||||
appExporter.SetStopGuard(exportStopGuard{g: appStopGuard})
|
||||
busyMark.AddProbe("an app export or import is running", appExporter.IsRunning)
|
||||
go busyMark.Run(ctx)
|
||||
// R-1279 (D48 for the import): the import downloads first, with the restores' helper and sentences.
|
||||
if backupMgr != nil {
|
||||
appExporter.SetImagePreparer(backupMgr.PrepareRestoreImages)
|
||||
@@ -2127,6 +2140,7 @@ func main() {
|
||||
|
||||
// --- Initialize web server ---
|
||||
webServer := web.NewServer(cfg, stackMgr, cpuCollector, backupMgr, sched, sett, alertMgr, notifier, updater, logger, Version)
|
||||
go webServer.WatchOperatorReboot(ctx) // D65: „Felhom support restarted your box at …" after an operator reboot
|
||||
// v0.316.0: which BUILD this is, not just which version it calls itself. Every shell's stylesheet
|
||||
// URL is derived from it, so two builds of one version can never share a cache entry. Init-only,
|
||||
// before the HTTP server starts.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// R-1350 (measured 2026-10-29 14:57 on Tester 1): the operator cleared gokapi's crash-loop hold from the hub, the app
|
||||
// stayed stopped (the clear starts nothing), and 30 s later the dead-app scan sent „Telepített alkalmazás nem fut:
|
||||
// Gokapi" — the hold had kept the app out of the scan, and the household's recorded wish was still „running".
|
||||
// The fix: the clear records that the stop is WANTED (desired state „stopped") for an app that is not up, BEFORE the
|
||||
// hold is lifted; a later Start records „running" and ends it.
|
||||
|
||||
// The consequence, through the scan's own derivation: a stopped app whose wish is „running" is DOWN (the alarm); the
|
||||
// same app with the wish „stopped" is not.
|
||||
func TestR1350_AStopWantedIsNotDown(t *testing.T) {
|
||||
st := func(desired string) []stacks.Stack {
|
||||
return []stacks.Stack{{Name: "gokapi", Deployed: true, State: stacks.StateStopped, AppConfig: &stacks.AppConfig{DesiredState: desired}}}
|
||||
}
|
||||
if _, states := classifyRunStates(st(stacks.DesiredStateRunning), nil, nil, time.Now()); len(states) != 1 || !states[0].Down {
|
||||
t.Fatalf("fixture: a stopped app wished running must be down: %+v", states)
|
||||
}
|
||||
if _, states := classifyRunStates(st(stacks.DesiredStateStopped), nil, nil, time.Now()); len(states) != 1 || states[0].Down {
|
||||
t.Fatalf("a stopped app wished stopped is down — the alarm fires: %+v", states)
|
||||
}
|
||||
}
|
||||
|
||||
// The clear records the wish BEFORE the hold is lifted (a dead-app scan between the two would read „no hold, wished
|
||||
// running" and mail), and only for an app that IS held. RED-PROOF: drop the keepStopped call from the clear → FAILS
|
||||
// („not recorded"); call it after the lift → FAILS („after the hold was lifted").
|
||||
func TestR1350_TheClearRecordsTheStopIsWanted(t *testing.T) {
|
||||
r := newD64Rig(t)
|
||||
r.hold(t, settings.RestoreHold{Stack: "gokapi", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop", Trip: 1})
|
||||
var recorded []string
|
||||
var heldWhenRecorded bool
|
||||
r.holdDeps.keepStopped = func(stack string) (bool, error) {
|
||||
recorded = append(recorded, stack)
|
||||
_, heldWhenRecorded = r.sett.GetRestoreHold(stack)
|
||||
return true, nil
|
||||
}
|
||||
o := r.actions()
|
||||
o.Reconcile([]report.OperatorAction{{ID: 21, Action: report.OpClearHold, Arg: "gokapi"}})
|
||||
if res := onlyResult(t, o, 21); res.Outcome != report.OutcomeDone {
|
||||
t.Fatalf("result %+v", res)
|
||||
}
|
||||
if len(recorded) != 1 || recorded[0] != "gokapi" {
|
||||
t.Fatalf("the stop was not recorded as wanted: %v", recorded)
|
||||
}
|
||||
if !heldWhenRecorded {
|
||||
t.Fatal("the wish was recorded after the hold was lifted — a scan in between mails app_start_failed")
|
||||
}
|
||||
if lg := r.logBuf.String(); !strings.Contains(lg, "gokapi: the stop is recorded as wanted") {
|
||||
t.Fatalf("the box's log must say it:\n%s", lg)
|
||||
}
|
||||
// a refusal (no hold) records nothing — a down app with no hold is a real fault the alarm must keep
|
||||
o.Reconcile([]report.OperatorAction{{ID: 22, Action: report.OpClearHold, Arg: "kimai"}})
|
||||
if res := onlyResult(t, o, 22); res.Outcome != report.OutcomeRefused {
|
||||
t.Fatalf("result %+v", res)
|
||||
}
|
||||
if len(recorded) != 1 {
|
||||
t.Fatalf("a refused clear recorded a wish: %v", recorded)
|
||||
}
|
||||
}
|
||||
|
||||
// The running controller's clear is wired with the record. RED-PROOF: drop keepStopped from main.go's holdDeps → FAILS.
|
||||
func TestR1350_MainWiresTheRecord(t *testing.T) {
|
||||
raw, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(raw), "keepStopped: stackMgr.RecordStopWantedIfDown") || !strings.Contains(string(raw), "clear: appStopGuard.ClearHoldByOperatorThen") {
|
||||
t.Fatal("main.go's hold clear does not record the wanted stop (R-1350)")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user