diff --git a/CHANGELOG.md b/CHANGELOG.md index 744bee7f..90399f43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ **MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0). +### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4) + +- `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier + install's set-aside unit (success, or „not started yet" — the data stays, D49 = A), the primary unit of the install the + restore REPLACED is set aside at once — moved whole (D47), one log line „replaced by the restored install", no + operator alarm. Before, the next backup's writer set it aside with a SECOND alarm and a second kept copy (measured on + Tester 1, 0.343.0, 17:36:54Z). A unit already of this install is left alone; a failed move leaves it for the next + writer's guard, as before. `TestR1288_ARestoreOfASetAsideUnitRetiresTheReplacedUnitQuietly` (the Tester 1 order; the + fake provider now adopts the unit's install id as the stack manager does). + +### The household's „later" media restart survives a controller restart (R-1282, P3) + +- `internal/web/media_events.go`: the 03:30 time is stored (`settings.media_restart_later_at`); `ResumeMediaLater` + (main, at start) re-arms it, runs it at once when it passed while the controller was down, and clears it when it ran — + once. `r1282_media_later_test.go`. + +Red-proofs: felhom.eu `documentation/audits/night-2026-10-25/redproof-r1288-r1282.txt`. + ### The line hold holds every seeder, forgets a removed app, and puts back the slot it held (R-1261, P3) - **Every running seeder is held** (`internal/upwatch/linehold.go` `SeedLineTestNow`): qBittorrent AND Transmission — diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 31d0b6a7..a6bdc95b 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -2124,6 +2124,7 @@ func main() { stackMgr.SetMigrationDoneHook(webServer.OnMigrationDone) // R-1024 (v0.313.0): a media app that comes up gets every present drive wired in (debounced, idempotent). stackMgr.SetOnMediaUp(webServer.RunMediaWiringSoon) + webServer.ResumeMediaLater() // R-1282: a „later" media restart survives a controller restart // R-536: „Alkalmazás telepítve" is sent when the async deploy ACTUALLY finishes, and a deploy // that ends badly now says so instead of being silence. The accept-time event is // `app_deploy_started`, emitted by the API router beside its 202. diff --git a/controller/internal/backup/recovery_unit_test.go b/controller/internal/backup/recovery_unit_test.go index 25958f5a..b023ef42 100644 --- a/controller/internal/backup/recovery_unit_test.go +++ b/controller/internal/backup/recovery_unit_test.go @@ -59,9 +59,17 @@ func (f *fakeRecoveryProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind func (f *fakeRecoveryProvider) RecoverStackSecrets(string, []string) map[string]string { return f.secrets } -func (f *fakeRecoveryProvider) RecreateStackDefinitionFromUnit(_, _ string, fullEnv map[string]string) error { +func (f *fakeRecoveryProvider) RecreateStackDefinitionFromUnit(_, composeDir string, fullEnv map[string]string) error { f.gotEnv = fullEnv f.calls = append(f.calls, "recreate") + // As the stack manager does (stacks/deploy.go, D47): the restored app takes the unit's install identity. + if b, err := os.ReadFile(filepath.Join(composeDir, "app.yaml")); err == nil { + for _, l := range strings.Split(string(b), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(l), "install_id:"); ok && strings.TrimSpace(v) != "" { + f.info.InstallID = strings.Trim(strings.TrimSpace(v), `"'`) + } + } + } return nil } func (f *fakeRecoveryProvider) StartStackServices(_ string, services []string) error { diff --git a/controller/internal/backup/unit_owner.go b/controller/internal/backup/unit_owner.go index ae47baa4..9673c253 100644 --- a/controller/internal/backup/unit_owner.go +++ b/controller/internal/backup/unit_owner.go @@ -323,5 +323,39 @@ func (m *Manager) RestoreSetAsideUnit(id string) (string, UnitRestoreResult, err } m.logger.Printf("[INFO] [backup] restore %s from the set-aside unit %s (an earlier install, D47)", k.Source, k.Path) res, err := m.RestoreFromRecoveryUnitAt(k.Source, k.Path) + if err == nil || errors.Is(err, ErrRestoreNotStarted) { + // The restored data stays in both cases (D49 = A), so the app is now the set-aside unit's install. + m.retireReplacedUnit(k.Source) + } return k.Source, res, err } + +// retireReplacedUnit (R-1288): after the household restored an earlier install's set-aside unit, the app's primary +// unit belongs to the install the restore REPLACED. It is set aside here, at the restore — moved whole, never deleted +// (D47), one log line and NO operator alarm: the household chose this restore, so nothing unexpected happened. Before, +// the next backup's writer found it and set it aside with a second alarm and a second kept copy that looked like a +// reinstall. A unit that is already this install's (or none) is left alone; a move that fails leaves it for the next +// writer, whose guard sets it aside as before. +func (m *Manager) retireReplacedUnit(stack string) { + nsRoot := m.namespaceRoot(m.GetAppDrivePath(stack)) + unitDir := RecoveryUnitPath(nsRoot, stack) + exists, unitID, unitSecrets := unitOwnerOf(unitDir) + if !exists { + return + } + liveID, liveSecrets, err := m.liveInstall(stack) + if err != nil { + m.logger.Printf("[WARN] [backup] %s: after the restore of an earlier install, this install's identity cannot be read (%v) — the replaced unit is left for the next backup's check (D47)", stack, err) + return + } + if v, _ := unitOwnerVerdict(unitID, liveID, unitSecrets, liveSecrets); v != ownerOther { + return + } + dest, err := m.setAsideUnit(stack, unitDir, nsRoot) + if err != nil { + m.logger.Printf("[WARN] [backup] %s: the unit of the install the restore replaced (%s) could not be set aside: %v — the next backup's check sets it aside (D47)", stack, orUnknown(unitID), err) + return + } + m.logger.Printf("[INFO] [backup] %s: the unit of the install the restore replaced (%s) is set aside whole → %s — replaced by the restored install %s; no alarm (R-1288, D47)", + stack, orUnknown(unitID), dest, orUnknown(liveID)) +} diff --git a/controller/internal/backup/unit_owner_test.go b/controller/internal/backup/unit_owner_test.go index 992a22a5..c05a8c3c 100644 --- a/controller/internal/backup/unit_owner_test.go +++ b/controller/internal/backup/unit_owner_test.go @@ -320,3 +320,56 @@ func TestD47_ASetAsideTellsTheOperatorOnce(t *testing.T) { t.Fatalf("the operator hears once, with both identities: %v", got) } } + +// R-1288, the Tester 1 order measured 2026-10-25 on 0.343.0: install A backed up → reinstall B → B's first backup sets +// A's unit aside (one alarm) and writes B's own unit → the household restores A's set-aside copy. Before: the next +// backup found B's unit and set it aside with a SECOND alarm. Now B's unit is set aside AT the restore, quietly, and +// the next backup writes A's unit with nothing more set aside. +// RED-PROOF: drop the retireReplacedUnit call → after the restore the primary place still holds B's unit, and the +// next capture alarms a second time. +func TestR1288_ARestoreOfASetAsideUnitRetiresTheReplacedUnitQuietly(t *testing.T) { + r := d47Setup(t, "aaaa000000000001") + var alarms []string + r.m.SetUnitSetAsideNotify(func(stack, unitID, liveID, dest string) { alarms = append(alarms, unitID) }) + r.reinstall("bbbb000000000002") + if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // B's first backup: A set aside, B's unit written + t.Fatal(err) + } + if len(alarms) != 1 || len(r.setAside(t)) != 1 { + t.Fatalf("precondition: one set-aside, one alarm; got alarms %v, set-aside %v", alarms, r.setAside(t)) + } + r.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { + return []DiscoveredDB{{StackName: "tandoor", ContainerName: "tandoor-postgres", DBType: DBTypePostgres}}, nil + } + r.m.importDBDump = func(context.Context, DiscoveredDB, string) error { return nil } + r.m.volumeReplayFrom = func(string, string) (int, error) { return 1, nil } + r.m.undoVolumeProbeFn = func(string) (int64, bool, error) { return 10, true, nil } + r.m.undoVolumeTarFn = func(vol, dir string) error { + return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("live"), 0o600) + } + ks := r.sett.GetKeptCopies() + if _, _, err := r.m.RestoreSetAsideUnit(ks[0].ID); err != nil { + t.Fatalf("restore of A's set-aside unit: %v", err) + } + if r.prov.info.InstallID != "aaaa000000000001" { + t.Fatalf("precondition: the restored app is install A, got %q", r.prov.info.InstallID) + } + if exists, id, _ := unitOwnerOf(r.unitDir()); exists { + t.Fatalf("after the restore the primary place still holds install %q's unit — it must be set aside at the restore", id) + } + if got := len(r.setAside(t)); got != 2 { + t.Fatalf("set-aside units after the restore: %d, want 2 (A's, kept as the household's copy, and B's)", got) + } + if len(alarms) != 1 { + t.Fatalf("the restore the household chose must not alarm: alarms %v", alarms) + } + if !strings.Contains(r.logs.String(), "replaced by the restored install") { + t.Fatal("the restore's set-aside must say why in one line") + } + if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // the next backup: A's own unit, nothing set aside + t.Fatal(err) + } + if len(alarms) != 1 || len(r.setAside(t)) != 2 { + t.Fatalf("the next backup set something aside again: alarms %v, set-aside %v", alarms, r.setAside(t)) + } +} diff --git a/controller/internal/settings/settings.go b/controller/internal/settings/settings.go index a47f64bc..c3d0ec50 100644 --- a/controller/internal/settings/settings.go +++ b/controller/internal/settings/settings.go @@ -83,6 +83,10 @@ type Settings struct { // measured upload line (the last 14 off-site pushes) and qBittorrent's default-limit record (who owns the limit). // Opaque on purpose — this package does not know the watch's types, and the watch does not reach into settings. Upwatch json.RawMessage `json:"upwatch,omitempty"` + + // MediaRestartLaterAt is the household's „later" media restart (RFC3339 UTC; empty = none) — stored so a controller + // restart before 03:30 does not forget it (R-1282). + MediaRestartLaterAt string `json:"media_restart_later_at,omitempty"` // UnhealthyStopEventSeeded (v0.269.0) guards the ONE-TIME add of app_stopped_unhealthy (the // seedAppUpdateEvents shape). UnhealthyStopEventSeeded bool `json:"unhealthy_stop_event_seeded,omitempty"` @@ -2929,6 +2933,29 @@ func (s *Settings) MarkMediaRestarted() error { return s.save() } +// GetMediaRestartLater returns the stored „later" media restart time (zero = none or unreadable). +func (s *Settings) GetMediaRestartLater() time.Time { + s.mu.RLock() + defer s.mu.RUnlock() + t, err := time.Parse(time.RFC3339, s.MediaRestartLaterAt) + if err != nil { + return time.Time{} + } + return t +} + +// SetMediaRestartLater stores (or, with a zero time, clears) the „later" media restart time and saves. +func (s *Settings) SetMediaRestartLater(at time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + if at.IsZero() { + s.MediaRestartLaterAt = "" + } else { + s.MediaRestartLaterAt = at.UTC().Format(time.RFC3339) + } + return s.save() +} + // MediaRestartPending reports whether a drive changed after the media apps last restarted. func (sp StoragePath) MediaRestartPending() bool { return sp.MediaChangedAt != "" && sp.MediaRestartedAt < sp.MediaChangedAt diff --git a/controller/internal/web/media_events.go b/controller/internal/web/media_events.go index a5af7266..4b908dfd 100644 --- a/controller/internal/web/media_events.go +++ b/controller/internal/web/media_events.go @@ -250,6 +250,53 @@ func nextMediaLater(now time.Time) time.Time { return t } +// mediaLaterRestart runs the „later" restart (tests replace it). +var mediaLaterRestart = func(s *Server, why string) { s.restartMediaApps(why) } + +// scheduleMediaLater arms the one „later" timer for at (a time already past fires at once); when it fires, the stored +// time is cleared, so it runs once. +func (s *Server) scheduleMediaLater(at time.Time) { + s.mediaMu.Lock() + defer s.mediaMu.Unlock() + if s.mediaLaterTmr != nil { + s.mediaLaterTmr.Stop() + } + s.mediaLaterAt = at + d := time.Until(at) + if d < 0 { + d = 0 + } + s.mediaLaterTmr = time.AfterFunc(d, func() { + mediaLaterRestart(s, "the household's „later\"") + s.mediaMu.Lock() + s.mediaLaterAt = time.Time{} + s.mediaMu.Unlock() + if s.settings != nil { + if err := s.settings.SetMediaRestartLater(time.Time{}); err != nil { + s.logger.Printf("[WARN] [media] clearing the stored „later\" media restart: %v", err) + } + } + }) +} + +// ResumeMediaLater (R-1282, main calls it at start) re-arms a „later" media restart the household asked for before +// this controller started: at its time, or at once when that time passed while the controller was down. It runs once. +func (s *Server) ResumeMediaLater() { + if s.settings == nil { + return + } + at := s.settings.GetMediaRestartLater() + if at.IsZero() { + return + } + if time.Until(at) <= 0 { + s.logger.Printf("[INFO] [media] the household's „later\" media restart (%s) passed while the controller was down — it runs now (R-1282)", at.Format(time.RFC3339)) + } else { + s.logger.Printf("[INFO] [media] the household's „later\" media restart is kept from before the restart: %s (R-1282)", at.Format(time.RFC3339)) + } + s.scheduleMediaLater(at) +} + // handleMediaRestart: POST /api/storage/media/restart {"when":"now"|"later","force":bool}. func (s *Server) handleMediaRestart(w http.ResponseWriter, r *http.Request) { var req struct { @@ -262,19 +309,13 @@ func (s *Server) handleMediaRestart(w http.ResponseWriter, r *http.Request) { } if req.When == "later" { at := nextMediaLater(time.Now()) - s.mediaMu.Lock() - if s.mediaLaterTmr != nil { - s.mediaLaterTmr.Stop() + if s.settings != nil { + if err := s.settings.SetMediaRestartLater(at); err != nil { + s.logger.Printf("[WARN] [media] storing the „later\" media restart time: %v — a controller restart before then forgets it", err) + } } - s.mediaLaterAt = at - s.mediaLaterTmr = time.AfterFunc(time.Until(at), func() { - s.restartMediaApps("the household's „later\"") - s.mediaMu.Lock() - s.mediaLaterAt = time.Time{} - s.mediaMu.Unlock() - }) - s.mediaMu.Unlock() - s.logger.Printf("[INFO] [media] media restart scheduled for %s (the household's „later\"; a controller restart forgets it, the notice stays)", at.Format(time.RFC3339)) + s.scheduleMediaLater(at) + s.logger.Printf("[INFO] [media] media restart scheduled for %s (the household's „later\"; stored — a controller restart keeps it, R-1282)", at.Format(time.RFC3339)) writeDiskJSON(w, http.StatusOK, true, "", map[string]string{"later_at": at.Format("2006-01-02 15:04")}) return } diff --git a/controller/internal/web/r1282_media_later_test.go b/controller/internal/web/r1282_media_later_test.go new file mode 100644 index 00000000..7942de65 --- /dev/null +++ b/controller/internal/web/r1282_media_later_test.go @@ -0,0 +1,79 @@ +package web + +import ( + "io" + "log" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// R-1282: the household's „later" media restart (03:30) was an in-memory timer — a controller restart before then +// (the night's self-update, a power cut) forgot it and the media apps never saw the new drive. The time is now stored; +// at start it is re-armed, a time that passed while the controller was down fires at once, and it runs ONCE. +// RED-PROOF: make ResumeMediaLater a no-op → the passed case never restarts the apps. +func r1282Server(t *testing.T) (*Server, *settings.Settings, *int32) { + t.Helper() + lg := log.New(io.Discard, "", 0) + sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), lg) + if err != nil { + t.Fatal(err) + } + var runs int32 + old := mediaLaterRestart + mediaLaterRestart = func(*Server, string) { atomic.AddInt32(&runs, 1) } + t.Cleanup(func() { mediaLaterRestart = old }) + return &Server{settings: sett, logger: lg}, sett, &runs +} + +func TestR1282_ALaterRestartThatPassedWhileDownRunsOnceAtStart(t *testing.T) { + s, sett, runs := r1282Server(t) + if err := sett.SetMediaRestartLater(time.Now().Add(-2 * time.Hour)); err != nil { // 03:30 passed while down + t.Fatal(err) + } + s.ResumeMediaLater() + deadline := time.Now().Add(2 * time.Second) + for atomic.LoadInt32(runs) == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if n := atomic.LoadInt32(runs); n != 1 { + t.Fatalf("the passed restart ran %d time(s) at start, want 1", n) + } + for !sett.GetMediaRestartLater().IsZero() && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if !sett.GetMediaRestartLater().IsZero() { + t.Fatal("the stored time must be cleared once it ran — else the next start runs it again") + } + // A second start finds nothing: it ran once. + s.ResumeMediaLater() + time.Sleep(50 * time.Millisecond) + if n := atomic.LoadInt32(runs); n != 1 { + t.Fatalf("a second start ran it again (%d)", n) + } +} + +func TestR1282_AFutureLaterRestartIsKeptOverAStart(t *testing.T) { + s, sett, runs := r1282Server(t) + at := time.Now().Add(1 * time.Hour) + if err := sett.SetMediaRestartLater(at); err != nil { + t.Fatal(err) + } + s.ResumeMediaLater() + time.Sleep(50 * time.Millisecond) + if n := atomic.LoadInt32(runs); n != 0 { + t.Fatalf("a restart due in an hour ran now (%d)", n) + } + s.mediaMu.Lock() + got, armed := s.mediaLaterAt, s.mediaLaterTmr != nil + s.mediaMu.Unlock() + if !armed || !got.Equal(at.UTC().Truncate(time.Second)) && got.Sub(at).Abs() > time.Second { + t.Fatalf("the timer must be armed for %v, got %v (armed %v)", at, got, armed) + } + if s.mediaLaterTmr != nil { + s.mediaLaterTmr.Stop() + } +}