controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)
gates / gates (push) Successful in 27s
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -78,6 +78,9 @@ type Settings struct {
|
||||
// AppUpdateEventsSeeded (v0.264.0) guards the ONE-TIME add of app_update_undone / app_update_held
|
||||
// into an existing household's stored prefs (seedAppUpdateEvents).
|
||||
AppUpdateEventsSeeded bool `json:"app_update_events_seeded,omitempty"`
|
||||
// UnhealthyStopEventSeeded (v0.269.0) guards the ONE-TIME add of app_stopped_unhealthy (the
|
||||
// seedAppUpdateEvents shape).
|
||||
UnhealthyStopEventSeeded bool `json:"unhealthy_stop_event_seeded,omitempty"`
|
||||
|
||||
// HubEscrowIdentityPresent (v0.199.0, R-204 item 4 / R-193) caches the report ACK's
|
||||
// `escrow.identity_blob_present` — whether the HUB is holding a sealed recovery package for this
|
||||
@@ -201,6 +204,10 @@ type Settings struct {
|
||||
// leaving the marker active would have Recover() start the broken app at the next controller
|
||||
// boot, hours later and quietly, which is the outcome the hold exists to prevent.
|
||||
RestoreHolds map[string]RestoreHold `json:"restore_holds,omitempty"`
|
||||
// UnhealthyStops (v0.269.0, decision 28) — when the box last stopped each app for a crash loop or an
|
||||
// out-of-memory storm (RFC3339). Survives the hold being lifted by Start, so a second stop within
|
||||
// 24 h is known to be a second one.
|
||||
UnhealthyStops map[string]string `json:"unhealthy_stops,omitempty"`
|
||||
|
||||
// Cross-drive restic repo password (auto-generated on first use)
|
||||
CrossDriveResticPassword string `json:"cross_drive_restic_password,omitempty"`
|
||||
@@ -611,6 +618,8 @@ var DefaultEnabledEvents = []string{
|
||||
// (or its undo) failed and the app is held. On by default; seeded into existing prefs once below.
|
||||
"app_update_undone",
|
||||
"app_update_held",
|
||||
// v0.269.0 (`09` §3 decision 28): the box stopped an app that kept crashing / ran out of memory.
|
||||
"app_stopped_unhealthy",
|
||||
}
|
||||
|
||||
// PendingEvent is an event queued for the next Hub push cycle.
|
||||
@@ -705,6 +714,7 @@ func Load(path string, logger *log.Logger) (*Settings, error) {
|
||||
s.migrateResticToRsync()
|
||||
s.seedOffboxEnlargeNotice()
|
||||
s.seedAppUpdateEvents()
|
||||
s.seedUnhealthyStopEvent()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -749,6 +759,25 @@ func (s *Settings) seedAppUpdateEvents() {
|
||||
}
|
||||
}
|
||||
|
||||
// seedUnhealthyStopEvent runs ONCE (v0.269.0, decision 28) — the seedAppUpdateEvents shape, ADD-ONLY:
|
||||
// a household whose stored prefs predate app_stopped_unhealthy gets it; a later opt-out sticks. The hub
|
||||
// (v0.123.0) runs the same one-time add on its side, because this list is pushed to it on every save.
|
||||
func (s *Settings) seedUnhealthyStopEvent() {
|
||||
if s.UnhealthyStopEventSeeded {
|
||||
return
|
||||
}
|
||||
s.UnhealthyStopEventSeeded = true
|
||||
if s.Notifications != nil && s.Notifications.EnabledEvents != nil {
|
||||
s.Notifications.EnabledEvents = appendIfAbsent(s.Notifications.EnabledEvents, "app_stopped_unhealthy")
|
||||
if s.log != nil {
|
||||
s.log.Printf("[INFO] [settings] app_stopped_unhealthy added to the household's notification prefs (one-time, add-only)")
|
||||
}
|
||||
}
|
||||
if err := s.save(); err != nil && s.log != nil {
|
||||
s.log.Printf("[ERROR] [settings] Failed to save the unhealthy-stop event seed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// migrateResticToRsync converts any cross-drive backup configs using restic to rsync.
|
||||
// Called once during Load() before the mutex is exposed.
|
||||
func (s *Settings) migrateResticToRsync() {
|
||||
@@ -1758,12 +1787,19 @@ type RestoreHold struct {
|
||||
// says support is informed. CopiesSeen lists what WAS there ("tier N at RFC3339"), for support.
|
||||
NoWholeCopy bool `json:"no_whole_copy,omitempty"`
|
||||
CopiesSeen []string `json:"copies_seen,omitempty"`
|
||||
// UnhealthyKind / Trip (v0.269.0, decision 28): "crash_loop" or "oom_storm", and whether this is the
|
||||
// first stop or a repeat within 24 h (Trip >= 2 → the sentence says support is informed).
|
||||
UnhealthyKind string `json:"unhealthy_kind,omitempty"`
|
||||
Trip int `json:"trip,omitempty"`
|
||||
}
|
||||
|
||||
// Hold reasons. See RestoreHold.Reason.
|
||||
const (
|
||||
HoldReasonRestoreFailed = "" // R-379/R-380: a restore AND its rollback failed
|
||||
HoldReasonUpdateFailed = "update_failed" // slice 4: the new version did not come up healthy
|
||||
// HoldReasonUnhealthyStop (v0.269.0, `09` §3 decision 28): the box stopped an app in a crash loop or
|
||||
// an out-of-memory storm. Lifted by the household's Start (one more try); nothing else starts it.
|
||||
HoldReasonUnhealthyStop = "unhealthy_stop"
|
||||
)
|
||||
|
||||
// SetRestoreHold records a hold. Modelled on SetDisconnected: a condition, plus what it is holding.
|
||||
@@ -2603,3 +2639,42 @@ func (s *Settings) OptOutRecoveryRemindersForEpoch() error {
|
||||
s.RecoveryRemindOptOutEpoch = s.RecoveryOfferEpoch
|
||||
return s.save()
|
||||
}
|
||||
|
||||
// LastUnhealthyStop returns when the box last stopped `stack` for being unhealthy (decision 28).
|
||||
func (s *Settings) LastUnhealthyStop(stack string) (time.Time, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
v, ok := s.UnhealthyStops[stack]
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
return t, err == nil
|
||||
}
|
||||
|
||||
// RecordUnhealthyStop stores the time of a stop (decision 28).
|
||||
func (s *Settings) RecordUnhealthyStop(stack string, at time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.UnhealthyStops == nil {
|
||||
s.UnhealthyStops = map[string]string{}
|
||||
}
|
||||
s.UnhealthyStops[stack] = at.UTC().Format(time.RFC3339)
|
||||
return s.save()
|
||||
}
|
||||
|
||||
// ClearUnhealthyStopHold lifts an app's hold ONLY when it is an unhealthy stop (the Start button's
|
||||
// one more try). Any other hold stays. Returns whether one was lifted.
|
||||
func (s *Settings) ClearUnhealthyStopHold(stack string) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
h, ok := s.RestoreHolds[stack]
|
||||
if !ok || h.Reason != HoldReasonUnhealthyStop {
|
||||
return false, nil
|
||||
}
|
||||
delete(s.RestoreHolds, stack)
|
||||
if s.log != nil {
|
||||
s.log.Printf("[INFO] [settings] unhealthy-stop hold LIFTED for %s (Start)", stack)
|
||||
}
|
||||
return true, s.save()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user