Operator actions in the report reply (D1, R-314/R-279/R-177, decision 185)
The hub may ask the running controller for a CLOSED list of actions, carried in the report ACK (operator_actions) and answered on the next report (operator_action_results): offsite_backup_now, abandon_stop, abandon_extend (1-30 days), run_job (fill-watch, offsite-integrity, offsite-proof, disk-health-check). Unknown action/job/argument -> refused, nothing called. Once per id (in memory; every action is safe to repeat). - internal/report/opactions.go: the executor; results re-sent until the hub stops listing the id. - scheduler.RunNow: refuses unknown / already-running jobs; OnDemand(ctx) makes an operator's offsite-integrity run even when not due. - ExtendAbandon never shortens the countdown and refuses in the hub phase; StopAbandon reports failure when the hub cancel failed (it said success). - Report ACK read cap 4 KiB -> 64 KiB (an ACK over the cap dropped every field in it). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -1077,6 +1077,15 @@ func main() {
|
||||
RecordEscrowKeyHash: sett.SetHubEscrowKeySHA256,
|
||||
Logger: logger,
|
||||
}
|
||||
// R-314/R-279/R-177 (`09` §3 decision 185): the operator's closed list of actions, carried in the
|
||||
// report reply and run in THIS process (opactions_wiring.go). The result rides the next report;
|
||||
// a finished result fires an out-of-cycle report through the trigger once it exists.
|
||||
opActions := report.NewOperatorActions(ctx, operatorActionHandlers(backupMgr, sched, func() {
|
||||
if f, ok := opActionReportFire.Load().(func()); ok && f != nil {
|
||||
f()
|
||||
}
|
||||
}, logger), logger)
|
||||
report.SetOperatorActions(opActions)
|
||||
// Wire hub verification: update settings when hub reports customer status
|
||||
hubPusher.OnPushResponse = func(resp *report.PushResponse) {
|
||||
if resp.CustomerBlocked {
|
||||
@@ -1120,6 +1129,8 @@ func main() {
|
||||
// generation). The web gate reads it on the next request — no restart needed.
|
||||
claimSync := &report.ClaimSync{Settings: sett, Logger: logger}
|
||||
claimSync.Reconcile(resp.Claim)
|
||||
// Decision 185: the operator's pending actions (each acted on once per id).
|
||||
opActions.Reconcile(resp.OperatorActions)
|
||||
}
|
||||
// Wire hub push status into alert manager for dashboard alerts
|
||||
alertMgr.SetHubPushStatus(func() web.HubPushStatusData {
|
||||
@@ -1496,7 +1507,9 @@ func main() {
|
||||
// collision costs one skipped day, not a missed check, because due-ness makes tomorrow try
|
||||
// again. What would be a real defect is a slot that collides EVERY night; this is not one.
|
||||
sched.Daily("offsite-integrity", "06:00", func(ctx context.Context) error {
|
||||
runOffsiteIntegrityCheck(ctx, backupMgr, notifier, logger, false)
|
||||
// Decision 185: an operator's run_job runs the check even when it is not due — the operator
|
||||
// asked for a check, not for „is a check due". The schedule still checks due-ness.
|
||||
runOffsiteIntegrityCheck(ctx, backupMgr, notifier, logger, scheduler.OnDemand(ctx))
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -1811,6 +1824,7 @@ func main() {
|
||||
}
|
||||
reportTrigger = report.NewTrigger(fireReport, logger)
|
||||
go reportTrigger.Run(ctx)
|
||||
opActionReportFire.Store(reportTrigger.Fire)
|
||||
|
||||
// Direction-2 immediate-sync (v0.140.0): hold a hanging GET against the hub's wait channel
|
||||
// and fire the trigger the instant operator intent moves — so an operator save round-trips in
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
|
||||
)
|
||||
|
||||
// opActionReportFire holds the out-of-cycle report trigger once main has built it (func()). The
|
||||
// executor is built earlier than the trigger, so it reads it through this holder.
|
||||
var opActionReportFire atomic.Value
|
||||
|
||||
// operatorActionHandlers wires the four operator actions (R-314/R-279/R-177, `09` §3 decision 185)
|
||||
// to the running process. ONE wiring site, called from main and from the wiring test, so the test
|
||||
// exercises the doors production uses (the seam-built-but-never-wired trap).
|
||||
//
|
||||
// offsite_backup_now makes the SAME four checks as the household's „back up now"
|
||||
// (offboxRunHandler) and then runs the same call, so an operator press can never do what the
|
||||
// household's own press cannot.
|
||||
func operatorActionHandlers(backupMgr *backup.Manager, sched *scheduler.Scheduler, resultReady func(), logger *log.Logger) report.OperatorActionHandlers {
|
||||
h := report.OperatorActionHandlers{ResultReady: resultReady}
|
||||
if sched != nil {
|
||||
h.RunJob = sched.RunNow
|
||||
}
|
||||
if backupMgr == nil {
|
||||
return h
|
||||
}
|
||||
h.OffsiteBackupNow = func() (string, func(ctx context.Context) error) {
|
||||
switch {
|
||||
case !backupMgr.OffboxConfigured():
|
||||
return "no off-site target is set on this box", nil
|
||||
case !backupMgr.OffboxRunnable():
|
||||
return "the off-site key step (escrow) is not done yet — the run is refused until it is", nil
|
||||
case backupMgr.OffboxOrphaned():
|
||||
return "the off-site store is orphaned (written with an older key) — a run cannot write to it", nil
|
||||
case backupMgr.IsRunning():
|
||||
return "a backup run is already in flight", nil
|
||||
}
|
||||
return "", func(ctx context.Context) error {
|
||||
if logger != nil {
|
||||
logger.Printf("[INFO] [opaction] off-site backup started at the operator's request")
|
||||
}
|
||||
err := backupMgr.RunOffboxBackupWithProgress(ctx)
|
||||
if errors.Is(err, backup.ErrOffboxRunInFlight) {
|
||||
return fmt.Errorf("%w: %v", report.ErrActionRefused, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
h.AbandonStop = backupMgr.StopAbandon
|
||||
h.AbandonExtend = func(days int) (time.Time, error) { return backupMgr.ExtendAbandon(days) }
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/report"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/scheduler"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// `09` §3 decision 185 (D1), the design's red test (1): a reply carrying abandon_stop during a
|
||||
// countdown stops it IN THE RUNNING MANAGER — and settings.json on disk agrees. This is the reason
|
||||
// the action rides the report reply instead of the CLI lever: the CLI runs a SECOND process, whose
|
||||
// write the running controller can overwrite (the lost update, main.go's R-241 block).
|
||||
func TestD1_AbandonStopActsOnTheRunningManagerAndDisk(t *testing.T) {
|
||||
logger := log.New(io.Discard, "", 0)
|
||||
dir := t.TempDir()
|
||||
settingsPath := filepath.Join(dir, "settings.json")
|
||||
sett, err := settings.Load(settingsPath, logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = dir
|
||||
cfg.Paths.SystemDataPath = filepath.Join(dir, "sys")
|
||||
mgr := backup.NewManager(cfg, sett, logger)
|
||||
// No test reaches a real ssh or restic (R-488 / R-650).
|
||||
mgr.SetOffboxSSH(func(context.Context, string, string, int, string, string, string) ([]byte, error) {
|
||||
t.Fatal("abandon_stop must issue no remote command")
|
||||
return nil, nil
|
||||
})
|
||||
mgr.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) {
|
||||
t.Fatal("abandon_stop must run no restic")
|
||||
return nil, nil
|
||||
})
|
||||
due := time.Now().Add(10 * 24 * time.Hour).UTC()
|
||||
if err := sett.SetOffboxTarget(&settings.OffboxTarget{
|
||||
Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo", Schedule: "daily",
|
||||
EscrowState: "escrowed",
|
||||
AbandonRepoPath: "/srv/repo.orphaned-20261001",
|
||||
AbandonStartedAt: time.Now().Add(-4 * 24 * time.Hour).UTC().Format(time.RFC3339),
|
||||
AbandonAt: due.Format(time.RFC3339),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !mgr.AbandonStatus().Active {
|
||||
t.Fatal("fixture: a countdown must be running")
|
||||
}
|
||||
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpAbandonStop}})
|
||||
|
||||
if mgr.AbandonStatus().Active {
|
||||
t.Fatal("the running manager still counts down after abandon_stop")
|
||||
}
|
||||
onDisk, err := settings.Load(settingsPath, logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tg := onDisk.GetOffboxTarget(); tg == nil || tg.AbandonAt != "" || tg.AbandonRepoPath == "" {
|
||||
t.Fatalf("settings.json disagrees: %+v (want no due date, the set-aside path kept)", tg)
|
||||
}
|
||||
rs := o.Results()
|
||||
if len(rs) != 1 || rs[0].Outcome != report.OutcomeDone {
|
||||
t.Fatalf("result = %+v", rs)
|
||||
}
|
||||
|
||||
// Control: the same countdown, a second press → nothing to stop, reported as such (not „done").
|
||||
o.Reconcile([]report.OperatorAction{{ID: 2, Action: report.OpAbandonStop}})
|
||||
if r := o.Results(); len(r) != 1 || r[0].ID != 2 || r[0].Outcome != report.OutcomeFailed {
|
||||
t.Fatalf("second stop: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// offsite_backup_now refuses exactly where the household's „back up now" refuses: no target here.
|
||||
func TestD1_OffsiteBackupNowRefusesWithoutATarget(t *testing.T) {
|
||||
logger := log.New(io.Discard, "", 0)
|
||||
dir := t.TempDir()
|
||||
sett, err := settings.Load(filepath.Join(dir, "settings.json"), logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = dir
|
||||
mgr := backup.NewManager(cfg, sett, logger)
|
||||
mgr.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) {
|
||||
t.Fatal("no restic may run")
|
||||
return nil, nil
|
||||
})
|
||||
o := report.NewOperatorActions(context.Background(), operatorActionHandlers(mgr, nil, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 9, Action: report.OpOffsiteBackupNow}})
|
||||
if r := o.Results(); len(r) != 1 || r[0].Outcome != report.OutcomeRefused {
|
||||
t.Fatalf("result = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// run_job reaches the scheduler's RunNow: a registered job runs; a job not registered on this box is
|
||||
// refused.
|
||||
func TestD1_RunJobUsesTheScheduler(t *testing.T) {
|
||||
logger := log.New(io.Discard, "", 0)
|
||||
sched := scheduler.New(logger)
|
||||
ran := make(chan bool, 1)
|
||||
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { ran <- scheduler.OnDemand(ctx); return nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sched.Start(ctx)
|
||||
defer sched.Stop()
|
||||
o := report.NewOperatorActions(ctx, operatorActionHandlers(nil, sched, nil, logger), logger)
|
||||
o.Reconcile([]report.OperatorAction{{ID: 1, Action: report.OpRunJob, Arg: "fill-watch"}, {ID: 2, Action: report.OpRunJob, Arg: "offsite-proof"}})
|
||||
select {
|
||||
case od := <-ran:
|
||||
if !od {
|
||||
t.Fatal("the job did not see itself as on-demand")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("fill-watch never ran")
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for len(o.Results()) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
got := map[int64]string{}
|
||||
for _, r := range o.Results() {
|
||||
got[r.ID] = r.Outcome
|
||||
}
|
||||
if got[1] != report.OutcomeDone || got[2] != report.OutcomeRefused {
|
||||
t.Fatalf("outcomes = %v, want 1:done 2:refused (offsite-proof not registered here)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// main.go hands the reply's list to the executor and the trigger to its holder — one call each
|
||||
// (the seam-built-but-never-wired trap: a green executor that main never calls proves nothing).
|
||||
func TestD1_MainWiresTheExecutor(t *testing.T) {
|
||||
f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := map[string]int{}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch fn := call.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := fn.X.(*ast.Ident); ok {
|
||||
found[x.Name+"."+fn.Sel.Name]++
|
||||
}
|
||||
case *ast.Ident:
|
||||
found[fn.Name]++
|
||||
}
|
||||
return true
|
||||
})
|
||||
for _, want := range []string{"opActions.Reconcile", "report.SetOperatorActions", "operatorActionHandlers", "opActionReportFire.Store"} {
|
||||
if found[want] != 1 {
|
||||
t.Errorf("main.go calls %s %d time(s), want exactly 1", want, found[want])
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user