controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)
gates / gates (push) Successful in 27s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 12:18:39 +02:00
parent 7c3b3a9694
commit 3c6b49b31c
141 changed files with 3401 additions and 237 deletions
+75
View File
@@ -78,6 +78,9 @@ type Settings struct {
// AppUpdateEventsSeeded (v0.264.0) guards the ONE-TIME add of app_update_undone / app_update_held
// into an existing household's stored prefs (seedAppUpdateEvents).
AppUpdateEventsSeeded bool `json:"app_update_events_seeded,omitempty"`
// UnhealthyStopEventSeeded (v0.269.0) guards the ONE-TIME add of app_stopped_unhealthy (the
// seedAppUpdateEvents shape).
UnhealthyStopEventSeeded bool `json:"unhealthy_stop_event_seeded,omitempty"`
// HubEscrowIdentityPresent (v0.199.0, R-204 item 4 / R-193) caches the report ACK's
// `escrow.identity_blob_present` — whether the HUB is holding a sealed recovery package for this
@@ -201,6 +204,10 @@ type Settings struct {
// leaving the marker active would have Recover() start the broken app at the next controller
// boot, hours later and quietly, which is the outcome the hold exists to prevent.
RestoreHolds map[string]RestoreHold `json:"restore_holds,omitempty"`
// UnhealthyStops (v0.269.0, decision 28) — when the box last stopped each app for a crash loop or an
// out-of-memory storm (RFC3339). Survives the hold being lifted by Start, so a second stop within
// 24 h is known to be a second one.
UnhealthyStops map[string]string `json:"unhealthy_stops,omitempty"`
// Cross-drive restic repo password (auto-generated on first use)
CrossDriveResticPassword string `json:"cross_drive_restic_password,omitempty"`
@@ -611,6 +618,8 @@ var DefaultEnabledEvents = []string{
// (or its undo) failed and the app is held. On by default; seeded into existing prefs once below.
"app_update_undone",
"app_update_held",
// v0.269.0 (`09` §3 decision 28): the box stopped an app that kept crashing / ran out of memory.
"app_stopped_unhealthy",
}
// PendingEvent is an event queued for the next Hub push cycle.
@@ -705,6 +714,7 @@ func Load(path string, logger *log.Logger) (*Settings, error) {
s.migrateResticToRsync()
s.seedOffboxEnlargeNotice()
s.seedAppUpdateEvents()
s.seedUnhealthyStopEvent()
return s, nil
}
@@ -749,6 +759,25 @@ func (s *Settings) seedAppUpdateEvents() {
}
}
// seedUnhealthyStopEvent runs ONCE (v0.269.0, decision 28) — the seedAppUpdateEvents shape, ADD-ONLY:
// a household whose stored prefs predate app_stopped_unhealthy gets it; a later opt-out sticks. The hub
// (v0.123.0) runs the same one-time add on its side, because this list is pushed to it on every save.
func (s *Settings) seedUnhealthyStopEvent() {
if s.UnhealthyStopEventSeeded {
return
}
s.UnhealthyStopEventSeeded = true
if s.Notifications != nil && s.Notifications.EnabledEvents != nil {
s.Notifications.EnabledEvents = appendIfAbsent(s.Notifications.EnabledEvents, "app_stopped_unhealthy")
if s.log != nil {
s.log.Printf("[INFO] [settings] app_stopped_unhealthy added to the household's notification prefs (one-time, add-only)")
}
}
if err := s.save(); err != nil && s.log != nil {
s.log.Printf("[ERROR] [settings] Failed to save the unhealthy-stop event seed: %v", err)
}
}
// migrateResticToRsync converts any cross-drive backup configs using restic to rsync.
// Called once during Load() before the mutex is exposed.
func (s *Settings) migrateResticToRsync() {
@@ -1758,12 +1787,19 @@ type RestoreHold struct {
// says support is informed. CopiesSeen lists what WAS there ("tier N at RFC3339"), for support.
NoWholeCopy bool `json:"no_whole_copy,omitempty"`
CopiesSeen []string `json:"copies_seen,omitempty"`
// UnhealthyKind / Trip (v0.269.0, decision 28): "crash_loop" or "oom_storm", and whether this is the
// first stop or a repeat within 24 h (Trip >= 2 → the sentence says support is informed).
UnhealthyKind string `json:"unhealthy_kind,omitempty"`
Trip int `json:"trip,omitempty"`
}
// Hold reasons. See RestoreHold.Reason.
const (
HoldReasonRestoreFailed = "" // R-379/R-380: a restore AND its rollback failed
HoldReasonUpdateFailed = "update_failed" // slice 4: the new version did not come up healthy
// HoldReasonUnhealthyStop (v0.269.0, `09` §3 decision 28): the box stopped an app in a crash loop or
// an out-of-memory storm. Lifted by the household's Start (one more try); nothing else starts it.
HoldReasonUnhealthyStop = "unhealthy_stop"
)
// SetRestoreHold records a hold. Modelled on SetDisconnected: a condition, plus what it is holding.
@@ -2603,3 +2639,42 @@ func (s *Settings) OptOutRecoveryRemindersForEpoch() error {
s.RecoveryRemindOptOutEpoch = s.RecoveryOfferEpoch
return s.save()
}
// LastUnhealthyStop returns when the box last stopped `stack` for being unhealthy (decision 28).
func (s *Settings) LastUnhealthyStop(stack string) (time.Time, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
v, ok := s.UnhealthyStops[stack]
if !ok {
return time.Time{}, false
}
t, err := time.Parse(time.RFC3339, v)
return t, err == nil
}
// RecordUnhealthyStop stores the time of a stop (decision 28).
func (s *Settings) RecordUnhealthyStop(stack string, at time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.UnhealthyStops == nil {
s.UnhealthyStops = map[string]string{}
}
s.UnhealthyStops[stack] = at.UTC().Format(time.RFC3339)
return s.save()
}
// ClearUnhealthyStopHold lifts an app's hold ONLY when it is an unhealthy stop (the Start button's
// one more try). Any other hold stays. Returns whether one was lifted.
func (s *Settings) ClearUnhealthyStopHold(stack string) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
h, ok := s.RestoreHolds[stack]
if !ok || h.Reason != HoldReasonUnhealthyStop {
return false, nil
}
delete(s.RestoreHolds, stack)
if s.log != nil {
s.log.Printf("[INFO] [settings] unhealthy-stop hold LIFTED for %s (Start)", stack)
}
return true, s.save()
}