R-1288: a restore of a set-aside unit sets the replaced install's unit aside at once, with no alarm. R-1282: the later media restart time is stored and resumed at start, once. Red-proved. Commit only.
gates / gates (push) Successful in 1m26s

This commit is contained in:
2026-10-25 21:37:55 +01:00
parent a79a292d72
commit b5863a96eb
8 changed files with 274 additions and 13 deletions
+1
View File
@@ -2124,6 +2124,7 @@ func main() {
stackMgr.SetMigrationDoneHook(webServer.OnMigrationDone)
// R-1024 (v0.313.0): a media app that comes up gets every present drive wired in (debounced, idempotent).
stackMgr.SetOnMediaUp(webServer.RunMediaWiringSoon)
webServer.ResumeMediaLater() // R-1282: a „later" media restart survives a controller restart
// R-536: „Alkalmazás telepítve" is sent when the async deploy ACTUALLY finishes, and a deploy
// that ends badly now says so instead of being silence. The accept-time event is
// `app_deploy_started`, emitted by the API router beside its 202.
@@ -59,9 +59,17 @@ func (f *fakeRecoveryProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind
func (f *fakeRecoveryProvider) RecoverStackSecrets(string, []string) map[string]string {
return f.secrets
}
func (f *fakeRecoveryProvider) RecreateStackDefinitionFromUnit(_, _ string, fullEnv map[string]string) error {
func (f *fakeRecoveryProvider) RecreateStackDefinitionFromUnit(_, composeDir string, fullEnv map[string]string) error {
f.gotEnv = fullEnv
f.calls = append(f.calls, "recreate")
// As the stack manager does (stacks/deploy.go, D47): the restored app takes the unit's install identity.
if b, err := os.ReadFile(filepath.Join(composeDir, "app.yaml")); err == nil {
for _, l := range strings.Split(string(b), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(l), "install_id:"); ok && strings.TrimSpace(v) != "" {
f.info.InstallID = strings.Trim(strings.TrimSpace(v), `"'`)
}
}
}
return nil
}
func (f *fakeRecoveryProvider) StartStackServices(_ string, services []string) error {
+34
View File
@@ -323,5 +323,39 @@ func (m *Manager) RestoreSetAsideUnit(id string) (string, UnitRestoreResult, err
}
m.logger.Printf("[INFO] [backup] restore %s from the set-aside unit %s (an earlier install, D47)", k.Source, k.Path)
res, err := m.RestoreFromRecoveryUnitAt(k.Source, k.Path)
if err == nil || errors.Is(err, ErrRestoreNotStarted) {
// The restored data stays in both cases (D49 = A), so the app is now the set-aside unit's install.
m.retireReplacedUnit(k.Source)
}
return k.Source, res, err
}
// retireReplacedUnit (R-1288): after the household restored an earlier install's set-aside unit, the app's primary
// unit belongs to the install the restore REPLACED. It is set aside here, at the restore — moved whole, never deleted
// (D47), one log line and NO operator alarm: the household chose this restore, so nothing unexpected happened. Before,
// the next backup's writer found it and set it aside with a second alarm and a second kept copy that looked like a
// reinstall. A unit that is already this install's (or none) is left alone; a move that fails leaves it for the next
// writer, whose guard sets it aside as before.
func (m *Manager) retireReplacedUnit(stack string) {
nsRoot := m.namespaceRoot(m.GetAppDrivePath(stack))
unitDir := RecoveryUnitPath(nsRoot, stack)
exists, unitID, unitSecrets := unitOwnerOf(unitDir)
if !exists {
return
}
liveID, liveSecrets, err := m.liveInstall(stack)
if err != nil {
m.logger.Printf("[WARN] [backup] %s: after the restore of an earlier install, this install's identity cannot be read (%v) — the replaced unit is left for the next backup's check (D47)", stack, err)
return
}
if v, _ := unitOwnerVerdict(unitID, liveID, unitSecrets, liveSecrets); v != ownerOther {
return
}
dest, err := m.setAsideUnit(stack, unitDir, nsRoot)
if err != nil {
m.logger.Printf("[WARN] [backup] %s: the unit of the install the restore replaced (%s) could not be set aside: %v — the next backup's check sets it aside (D47)", stack, orUnknown(unitID), err)
return
}
m.logger.Printf("[INFO] [backup] %s: the unit of the install the restore replaced (%s) is set aside whole → %s — replaced by the restored install %s; no alarm (R-1288, D47)",
stack, orUnknown(unitID), dest, orUnknown(liveID))
}
@@ -320,3 +320,56 @@ func TestD47_ASetAsideTellsTheOperatorOnce(t *testing.T) {
t.Fatalf("the operator hears once, with both identities: %v", got)
}
}
// R-1288, the Tester 1 order measured 2026-10-25 on 0.343.0: install A backed up → reinstall B → B's first backup sets
// A's unit aside (one alarm) and writes B's own unit → the household restores A's set-aside copy. Before: the next
// backup found B's unit and set it aside with a SECOND alarm. Now B's unit is set aside AT the restore, quietly, and
// the next backup writes A's unit with nothing more set aside.
// RED-PROOF: drop the retireReplacedUnit call → after the restore the primary place still holds B's unit, and the
// next capture alarms a second time.
func TestR1288_ARestoreOfASetAsideUnitRetiresTheReplacedUnitQuietly(t *testing.T) {
r := d47Setup(t, "aaaa000000000001")
var alarms []string
r.m.SetUnitSetAsideNotify(func(stack, unitID, liveID, dest string) { alarms = append(alarms, unitID) })
r.reinstall("bbbb000000000002")
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // B's first backup: A set aside, B's unit written
t.Fatal(err)
}
if len(alarms) != 1 || len(r.setAside(t)) != 1 {
t.Fatalf("precondition: one set-aside, one alarm; got alarms %v, set-aside %v", alarms, r.setAside(t))
}
r.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
return []DiscoveredDB{{StackName: "tandoor", ContainerName: "tandoor-postgres", DBType: DBTypePostgres}}, nil
}
r.m.importDBDump = func(context.Context, DiscoveredDB, string) error { return nil }
r.m.volumeReplayFrom = func(string, string) (int, error) { return 1, nil }
r.m.undoVolumeProbeFn = func(string) (int64, bool, error) { return 10, true, nil }
r.m.undoVolumeTarFn = func(vol, dir string) error {
return os.WriteFile(filepath.Join(dir, vol+".tar"), []byte("live"), 0o600)
}
ks := r.sett.GetKeptCopies()
if _, _, err := r.m.RestoreSetAsideUnit(ks[0].ID); err != nil {
t.Fatalf("restore of A's set-aside unit: %v", err)
}
if r.prov.info.InstallID != "aaaa000000000001" {
t.Fatalf("precondition: the restored app is install A, got %q", r.prov.info.InstallID)
}
if exists, id, _ := unitOwnerOf(r.unitDir()); exists {
t.Fatalf("after the restore the primary place still holds install %q's unit — it must be set aside at the restore", id)
}
if got := len(r.setAside(t)); got != 2 {
t.Fatalf("set-aside units after the restore: %d, want 2 (A's, kept as the household's copy, and B's)", got)
}
if len(alarms) != 1 {
t.Fatalf("the restore the household chose must not alarm: alarms %v", alarms)
}
if !strings.Contains(r.logs.String(), "replaced by the restored install") {
t.Fatal("the restore's set-aside must say why in one line")
}
if err := r.m.captureRecoveryUnit("tandoor", false); err != nil { // the next backup: A's own unit, nothing set aside
t.Fatal(err)
}
if len(alarms) != 1 || len(r.setAside(t)) != 2 {
t.Fatalf("the next backup set something aside again: alarms %v, set-aside %v", alarms, r.setAside(t))
}
}
+27
View File
@@ -83,6 +83,10 @@ type Settings struct {
// measured upload line (the last 14 off-site pushes) and qBittorrent's default-limit record (who owns the limit).
// Opaque on purpose — this package does not know the watch's types, and the watch does not reach into settings.
Upwatch json.RawMessage `json:"upwatch,omitempty"`
// MediaRestartLaterAt is the household's „later" media restart (RFC3339 UTC; empty = none) — stored so a controller
// restart before 03:30 does not forget it (R-1282).
MediaRestartLaterAt string `json:"media_restart_later_at,omitempty"`
// UnhealthyStopEventSeeded (v0.269.0) guards the ONE-TIME add of app_stopped_unhealthy (the
// seedAppUpdateEvents shape).
UnhealthyStopEventSeeded bool `json:"unhealthy_stop_event_seeded,omitempty"`
@@ -2929,6 +2933,29 @@ func (s *Settings) MarkMediaRestarted() error {
return s.save()
}
// GetMediaRestartLater returns the stored „later" media restart time (zero = none or unreadable).
func (s *Settings) GetMediaRestartLater() time.Time {
s.mu.RLock()
defer s.mu.RUnlock()
t, err := time.Parse(time.RFC3339, s.MediaRestartLaterAt)
if err != nil {
return time.Time{}
}
return t
}
// SetMediaRestartLater stores (or, with a zero time, clears) the „later" media restart time and saves.
func (s *Settings) SetMediaRestartLater(at time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
if at.IsZero() {
s.MediaRestartLaterAt = ""
} else {
s.MediaRestartLaterAt = at.UTC().Format(time.RFC3339)
}
return s.save()
}
// MediaRestartPending reports whether a drive changed after the media apps last restarted.
func (sp StoragePath) MediaRestartPending() bool {
return sp.MediaChangedAt != "" && sp.MediaRestartedAt < sp.MediaChangedAt
+53 -12
View File
@@ -250,6 +250,53 @@ func nextMediaLater(now time.Time) time.Time {
return t
}
// mediaLaterRestart runs the „later" restart (tests replace it).
var mediaLaterRestart = func(s *Server, why string) { s.restartMediaApps(why) }
// scheduleMediaLater arms the one „later" timer for at (a time already past fires at once); when it fires, the stored
// time is cleared, so it runs once.
func (s *Server) scheduleMediaLater(at time.Time) {
s.mediaMu.Lock()
defer s.mediaMu.Unlock()
if s.mediaLaterTmr != nil {
s.mediaLaterTmr.Stop()
}
s.mediaLaterAt = at
d := time.Until(at)
if d < 0 {
d = 0
}
s.mediaLaterTmr = time.AfterFunc(d, func() {
mediaLaterRestart(s, "the household's „later\"")
s.mediaMu.Lock()
s.mediaLaterAt = time.Time{}
s.mediaMu.Unlock()
if s.settings != nil {
if err := s.settings.SetMediaRestartLater(time.Time{}); err != nil {
s.logger.Printf("[WARN] [media] clearing the stored „later\" media restart: %v", err)
}
}
})
}
// ResumeMediaLater (R-1282, main calls it at start) re-arms a „later" media restart the household asked for before
// this controller started: at its time, or at once when that time passed while the controller was down. It runs once.
func (s *Server) ResumeMediaLater() {
if s.settings == nil {
return
}
at := s.settings.GetMediaRestartLater()
if at.IsZero() {
return
}
if time.Until(at) <= 0 {
s.logger.Printf("[INFO] [media] the household's „later\" media restart (%s) passed while the controller was down — it runs now (R-1282)", at.Format(time.RFC3339))
} else {
s.logger.Printf("[INFO] [media] the household's „later\" media restart is kept from before the restart: %s (R-1282)", at.Format(time.RFC3339))
}
s.scheduleMediaLater(at)
}
// handleMediaRestart: POST /api/storage/media/restart {"when":"now"|"later","force":bool}.
func (s *Server) handleMediaRestart(w http.ResponseWriter, r *http.Request) {
var req struct {
@@ -262,19 +309,13 @@ func (s *Server) handleMediaRestart(w http.ResponseWriter, r *http.Request) {
}
if req.When == "later" {
at := nextMediaLater(time.Now())
s.mediaMu.Lock()
if s.mediaLaterTmr != nil {
s.mediaLaterTmr.Stop()
if s.settings != nil {
if err := s.settings.SetMediaRestartLater(at); err != nil {
s.logger.Printf("[WARN] [media] storing the „later\" media restart time: %v — a controller restart before then forgets it", err)
}
}
s.mediaLaterAt = at
s.mediaLaterTmr = time.AfterFunc(time.Until(at), func() {
s.restartMediaApps("the household's „later\"")
s.mediaMu.Lock()
s.mediaLaterAt = time.Time{}
s.mediaMu.Unlock()
})
s.mediaMu.Unlock()
s.logger.Printf("[INFO] [media] media restart scheduled for %s (the household's „later\"; a controller restart forgets it, the notice stays)", at.Format(time.RFC3339))
s.scheduleMediaLater(at)
s.logger.Printf("[INFO] [media] media restart scheduled for %s (the household's „later\"; stored — a controller restart keeps it, R-1282)", at.Format(time.RFC3339))
writeDiskJSON(w, http.StatusOK, true, "", map[string]string{"later_at": at.Format("2006-01-02 15:04")})
return
}
@@ -0,0 +1,79 @@
package web
import (
"io"
"log"
"path/filepath"
"sync/atomic"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// R-1282: the household's „later" media restart (03:30) was an in-memory timer — a controller restart before then
// (the night's self-update, a power cut) forgot it and the media apps never saw the new drive. The time is now stored;
// at start it is re-armed, a time that passed while the controller was down fires at once, and it runs ONCE.
// RED-PROOF: make ResumeMediaLater a no-op → the passed case never restarts the apps.
func r1282Server(t *testing.T) (*Server, *settings.Settings, *int32) {
t.Helper()
lg := log.New(io.Discard, "", 0)
sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), lg)
if err != nil {
t.Fatal(err)
}
var runs int32
old := mediaLaterRestart
mediaLaterRestart = func(*Server, string) { atomic.AddInt32(&runs, 1) }
t.Cleanup(func() { mediaLaterRestart = old })
return &Server{settings: sett, logger: lg}, sett, &runs
}
func TestR1282_ALaterRestartThatPassedWhileDownRunsOnceAtStart(t *testing.T) {
s, sett, runs := r1282Server(t)
if err := sett.SetMediaRestartLater(time.Now().Add(-2 * time.Hour)); err != nil { // 03:30 passed while down
t.Fatal(err)
}
s.ResumeMediaLater()
deadline := time.Now().Add(2 * time.Second)
for atomic.LoadInt32(runs) == 0 && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if n := atomic.LoadInt32(runs); n != 1 {
t.Fatalf("the passed restart ran %d time(s) at start, want 1", n)
}
for !sett.GetMediaRestartLater().IsZero() && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if !sett.GetMediaRestartLater().IsZero() {
t.Fatal("the stored time must be cleared once it ran — else the next start runs it again")
}
// A second start finds nothing: it ran once.
s.ResumeMediaLater()
time.Sleep(50 * time.Millisecond)
if n := atomic.LoadInt32(runs); n != 1 {
t.Fatalf("a second start ran it again (%d)", n)
}
}
func TestR1282_AFutureLaterRestartIsKeptOverAStart(t *testing.T) {
s, sett, runs := r1282Server(t)
at := time.Now().Add(1 * time.Hour)
if err := sett.SetMediaRestartLater(at); err != nil {
t.Fatal(err)
}
s.ResumeMediaLater()
time.Sleep(50 * time.Millisecond)
if n := atomic.LoadInt32(runs); n != 0 {
t.Fatalf("a restart due in an hour ran now (%d)", n)
}
s.mediaMu.Lock()
got, armed := s.mediaLaterAt, s.mediaLaterTmr != nil
s.mediaMu.Unlock()
if !armed || !got.Equal(at.UTC().Truncate(time.Second)) && got.Sub(at).Abs() > time.Second {
t.Fatalf("the timer must be armed for %v, got %v (armed %v)", at, got, armed)
}
if s.mediaLaterTmr != nil {
s.mediaLaterTmr.Stop()
}
}