D48: a restore downloads first — every restore path resolves and pulls every image of the definition that will run before the stop, under the image-work lock held until the restore returns; a failed pull refuses with the app untouched (a deleted version and a busy registry read differently), one operator line. Red-proved. Commit only.
gates / gates (push) Successful in 1m26s

This commit is contained in:
2026-10-25 09:38:28 +01:00
parent b680db85cb
commit 2d71270858
16 changed files with 404 additions and 3 deletions
+5
View File
@@ -243,6 +243,11 @@ type Manager struct {
restorePutBackNotify func(stack string, replayErr error)
// restoreNotStartedNotify (R-1270) is called once when a restore's data is back but its app did not start in time.
restoreNotStartedNotify func(stack, kind string, cause error)
// D48: the restore's image probe and pull (nil → docker; tests set them, TestMain makes every image present), and
// the operator notice of a restore refused because an image could not be downloaded.
imagePresentFn func(ref string) bool
imagePullFn func(ctx context.Context, ref string) (string, error)
restorePullRefusedNotify func(stack, ref string, cause error)
// R-403 unit-REHYDRATE seam — the copy that refills the primary unit from the Tier-2 mirror after
// a restore, overridable so the "only when hollow" condition and the byte-identical non-effect are
+3
View File
@@ -19,6 +19,9 @@ func TestMain(m *testing.M) {
healthSettle = 0
healthInterval = 2 * time.Millisecond
healthTimeout = 50 * time.Millisecond
// D48 / R-650: no test in this package reaches Docker for the restore's pre-pull — every image is present unless a
// test says otherwise (m.imagePresentFn / m.imagePullFn).
imagePresentDefault = func(string) bool { return true }
os.Exit(m.Run())
}
@@ -946,6 +946,20 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
}
}
// D48: DOWNLOAD FIRST — every image of the definition that will run, before the stop, or a refusal with the app
// untouched. Held until this restore returns (R-863).
runCompose := filepath.Join(scratchCompose, "docker-compose.yml")
if !defineFromSnapshot {
if live, ok := m.stackProvider.GetStackComposePath(stack); ok && live != "" {
runCompose = live
}
}
releaseImages, pErr := m.prepareRestoreImages(stack, runCompose)
if pErr != nil {
return res, pErr
}
defer releaseImages()
// --- FILES ----------------------------------------------------------------------------------
// R-166: mark the stop→restore→start window BEFORE stopping. A controller killed anywhere inside
// it used to leave the app down with nothing on disk recording that it was owed a restart — and a
+9
View File
@@ -79,6 +79,15 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
}
}
// D48: the live definition's images are on the box before the stop, or the restore is refused untouched.
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok && composePath != "" {
releaseImages, pErr := m.prepareRestoreImages(stackName, composePath)
if pErr != nil {
return pErr
}
defer releaseImages()
}
// Stop the app before restore
if m.isDebug() {
m.logger.Printf("[DEBUG] RestoreApp: step 1/4 — stopping app %s", stackName)
+111
View File
@@ -0,0 +1,111 @@
package backup
import (
"context"
"errors"
"fmt"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// ── D48 (operator ruling 2026-10-25 08:38; `09` §3 decision 257): a restore downloads first ─────────────────────────
//
// BEFORE: every restore stopped the app and moved its data first; the first pull was inside the database-only start
// (`docker compose up -d <db>`), after the stop. A version that could not be downloaded then ended in the D22 put-back —
// the whole stop, copy and pour-back for nothing, and an app that was down the while.
//
// THE RULE. After the existing refusals and before the first stop or change, every image the definition that will RUN
// names is resolved: present on the box, or pulled now. Under the image-work lock held SHARED from the first check
// until the restore's last start (R-863: the image clean-up cannot delete a pulled image before the start — its pass
// takes the lock exclusively with TryLock and simply does not run meanwhile). A failed pull is a REFUSAL: the app is
// not stopped and its data is not moved. The household reads which version could not be downloaded, and why — a
// version its maker deleted (Docker answers „manifest unknown", „not found", „pull access denied") reads differently
// from a registry that is busy or unreachable (anything else, „toomanyrequests" and timeouts included).
// ErrRestoreImageUnavailable is the KIND of the refusal: an image of the backup's version could not be downloaded.
var ErrRestoreImageUnavailable = errors.New("an image of the backup's version cannot be downloaded")
// restorePullTimeout bounds one image pull of a restore.
const restorePullTimeout = 15 * time.Minute
// imagePresentDefault / imagePullDefault are the production probe and pull (tests replace them in TestMain, so no test
// in this package reaches Docker — R-650; a test that needs a missing image sets m.imagePresentFn / m.imagePullFn).
var (
imagePresentDefault = func(ref string) bool {
return dockerexec.Command("docker", "image", "inspect", "--format", "{{.Id}}", ref).Run() == nil
}
imagePullDefault = func(ctx context.Context, ref string) (string, error) {
out, err := dockerexec.CommandContext(ctx, "docker", "pull", ref).CombinedOutput()
return string(out), err
}
)
func (m *Manager) imagePresent(ref string) bool {
if m.imagePresentFn != nil {
return m.imagePresentFn(ref)
}
return imagePresentDefault(ref)
}
func (m *Manager) imagePull(ctx context.Context, ref string) (string, error) {
if m.imagePullFn != nil {
return m.imagePullFn(ctx, ref)
}
return imagePullDefault(ctx, ref)
}
// pullGone reports whether Docker's answer says the version does not exist (deleted by its maker, a wrong name) rather
// than that the registry could not be reached or is busy.
func pullGone(out string) bool {
low := strings.ToLower(out)
for _, s := range []string{"manifest unknown", "not found", "repository does not exist", "pull access denied", "name unknown", "no such manifest"} {
if strings.Contains(low, s) {
return true
}
}
return false
}
// prepareRestoreImages resolves every image composePath names and pulls the absent ones, holding the image-work lock
// until release(). err = the refusal (the household's sentence); the lock is released already then.
func (m *Manager) prepareRestoreImages(stack, composePath string) (release func(), err error) {
refs := ParseComposeImages(composePath)
end := dockerexec.HoldImageWork()
for _, ref := range refs {
if m.imagePresent(ref) {
continue
}
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), restorePullTimeout)
out, perr := m.imagePull(ctx, ref)
cancel()
if perr != nil {
end()
tail := strings.TrimSpace(out)
if len(tail) > 300 {
tail = tail[len(tail)-300:]
}
key := "err.backup.restore_image_unreachable"
why := "the registry did not answer or is busy"
if pullGone(out) {
key, why = "err.backup.restore_image_gone", "the version does not exist any more"
}
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s before anything was touched: the image %s could not be downloaded (%s; after %s): %v — %s (D48)",
stack, ref, why, time.Since(start).Round(time.Second), perr, tail)
if m.restorePullRefusedNotify != nil {
m.restorePullRefusedNotify(stack, ref, fmt.Errorf("%s: %v", why, perr))
}
return func() {}, util.MsgErrorf(ErrRestoreImageUnavailable, key, PinsVersion([]string{ref}))
}
m.logger.Printf("[INFO] [backup] Restore %s: downloaded %s before anything was touched (%s, D48)", stack, ref, time.Since(start).Round(time.Second))
}
return end, nil
}
// SetRestorePullRefusedNotify wires the operator notice for a restore refused because an image could not be downloaded.
func (m *Manager) SetRestorePullRefusedNotify(fn func(stack, ref string, cause error)) {
m.restorePullRefusedNotify = fn
}
@@ -0,0 +1,138 @@
package backup
import (
"context"
"errors"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
// D48 (operator ruling 2026-10-25; `09` §3 decision 257) — a restore downloads first. Before (`e1613ae`): the first pull
// was inside the database-only start, AFTER the stop; a version that could not be downloaded ended in the D22 put-back.
// COMPANION RED-PROOF (REPORT.md): with prepareRestoreImages not called, TestD48_AnUnpullablePinIsRefusedBeforeTheStop
// and TestD48_TheOffsiteRestoreToo fail — the app is stopped (the calls start with "stop").
func d48Unpullable(m *Manager, answer string) *[]string {
var pulled []string
m.imagePresentFn = func(string) bool { return false }
m.imagePullFn = func(_ context.Context, ref string) (string, error) {
pulled = append(pulled, ref)
return answer, errors.New("exit status 1")
}
return &pulled
}
// An unpullable pin: refused, the stop never called, nothing poured, no copy made, no definition written.
func TestD48_AnUnpullablePinIsRefusedBeforeTheStop(t *testing.T) {
m, p, rig, imported := r1177Fixture(t)
pulled := d48Unpullable(m, "Error response from daemon: manifest for example/app:9.9.9 not found: manifest unknown")
var notified []string
m.SetRestorePullRefusedNotify(func(stack, ref string, _ error) { notified = append(notified, stack+" "+ref) })
_, err := m.RestoreFromRecoveryUnit("app")
if !errors.Is(err, ErrRestoreImageUnavailable) || len(*pulled) == 0 {
t.Fatalf("an unpullable pin must refuse, got %v (pulled %v)", err, *pulled)
}
if want := util.MsgError("err.backup.restore_image_gone", PinsVersion([]string{(*pulled)[0]})).Error(); err.Error() != want {
t.Fatalf("a deleted version reads its own sentence, got %q", err.Error())
}
if p.stopped || len(p.calls) != 0 {
t.Fatalf("the app must not be stopped or touched at all, calls %v", p.calls)
}
if len(rig.poured) != 0 || len(rig.defsWrote) != 0 || len(*imported) != 0 || len(undoCopies(t, rig.undoRoot)) != 0 {
t.Fatalf("nothing may move: poured %v, defs %v, imported %v", rig.poured, rig.defsWrote, *imported)
}
if len(notified) != 1 {
t.Fatalf("the operator gets one notice, got %v", notified)
}
}
// A registry that is busy or unreachable reads differently from a deleted version.
func TestD48_AnUnreachableRegistryReadsDifferently(t *testing.T) {
m, p, _, _ := r1177Fixture(t)
pulled := d48Unpullable(m, "Error response from daemon: Get \"https://registry-1.docker.io/v2/\": dial tcp: lookup registry-1.docker.io: i/o timeout")
_, err := m.RestoreFromRecoveryUnit("app")
if len(*pulled) == 0 {
t.Fatalf("the restore never asked for the image (err %v)", err)
}
if want := util.MsgError("err.backup.restore_image_unreachable", PinsVersion([]string{(*pulled)[0]})).Error(); err == nil || err.Error() != want {
t.Fatalf("an unreachable registry reads „try again later\", got %v", err)
}
if p.stopped {
t.Fatal("the app must not be stopped")
}
}
// A pullable pin: pulled BEFORE the stop, and the restore runs as before.
func TestD48_APullablePinIsPulledFirstAndTheRestoreRuns(t *testing.T) {
m, p, _, imported := r1177Fixture(t)
m.imagePresentFn = func(string) bool { return false }
m.imagePullFn = func(_ context.Context, ref string) (string, error) {
p.calls = append(p.calls, "pull:"+ref)
return "Status: Downloaded newer image", nil
}
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
t.Fatalf("a pullable restore runs: %v", err)
}
if len(p.calls) == 0 || !strings.HasPrefix(p.calls[0], "pull:") {
t.Fatalf("the pull must come before the stop, calls %v", p.calls)
}
stop := -1
for i, c := range p.calls {
if c == "stop" {
stop = i
break
}
}
for i, c := range p.calls {
if strings.HasPrefix(c, "pull:") && i > stop {
t.Fatalf("every pull before the stop, calls %v", p.calls)
}
}
if len(*imported) != 1 {
t.Fatalf("the restore ran as before (one database replayed), got %v", *imported)
}
}
// The second drive's whole restore asks the unit half BEFORE its file half (R-1178): the pull is in that check.
func TestD48_TheCheckOnlyPassCarriesThePull(t *testing.T) {
m, p, _, _ := r1177Fixture(t)
d48Unpullable(m, "manifest unknown")
unit := m.primaryUnitDirFor("app")
if _, err := m.RestoreFromRecoveryUnitAtWith("app", unit, UnitRestoreOptions{CheckOnly: true}); !errors.Is(err, ErrRestoreImageUnavailable) {
t.Fatalf("the check must refuse an unpullable pin, got %v", err)
}
if p.stopped {
t.Fatal("the check never stops the app")
}
}
// The off-site restore: refused before the stop too.
func TestD48_TheOffsiteRestoreToo(t *testing.T) {
m, prov, imported := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
d48Unpullable(m, "pull access denied for example/immich, repository does not exist")
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
if !errors.Is(err, ErrRestoreImageUnavailable) {
t.Fatalf("an off-site restore with an unpullable image must refuse, got %v", err)
}
if len(prov.calls) != 0 || len(*imported) != 0 {
t.Fatalf("the app must not be stopped and nothing replayed: calls %v, imported %v", prov.calls, *imported)
}
}
// Docker's answers, as the two sentences read them.
func TestD48_PullGone(t *testing.T) {
for out, want := range map[string]bool{
"manifest for x:1 not found: manifest unknown": true,
"pull access denied for x, repository does not exist or may require": true,
"toomanyrequests: You have reached your pull rate limit": false,
"dial tcp: lookup registry-1.docker.io: i/o timeout": false,
"net/http: TLS handshake timeout": false,
} {
if got := pullGone(out); got != want {
t.Errorf("pullGone(%q) = %v, want %v", out, got, want)
}
}
}
@@ -463,6 +463,21 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
}
undoExists = ex
}
// D48 (`09` §3 decision 257): DOWNLOAD FIRST. Every image the definition that will run names is on the box before
// the stop — or the restore is refused here, with the app running and its data where it was. The lock is held
// until this function returns, past the last start (R-863). The CheckOnly pass carries it too (R-1178), so the
// second drive's file half never runs before a pull that would refuse.
runCompose := filepath.Join(composeDir, "docker-compose.yml")
if !fileExists(runCompose) {
if live, ok := m.stackProvider.GetStackComposePath(stackName); ok && live != "" {
runCompose = live
}
}
releaseImages, pErr := m.prepareRestoreImages(stackName, runCompose)
if pErr != nil {
return res, pErr
}
defer releaseImages()
if opt.CheckOnly {
return res, nil
}
@@ -420,6 +420,15 @@ func (m *Manager) RestoreTier2Files(stackName string) (filesRestored int, err er
}
m.logger.Printf("[INFO] [backup] Tier-2 file restore for %s: %s (v2) → %s (additive-only)", stackName, destBase, liveNsRoot)
// D48: the live definition's images are on the box before the stop, or the restore is refused untouched.
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok && composePath != "" {
releaseImages, pErr := m.prepareRestoreImages(stackName, composePath)
if pErr != nil {
return 0, pErr
}
defer releaseImages()
}
// Stop → copy → start → health (the standard restore shape; F17: errors surface, never swallowed).
if stopErr := m.stackProvider.StopStack(stackName); stopErr != nil {
m.logger.Printf("[WARN] [backup] could not stop %s before Tier-2 file restore: %v (continuing)", stackName, stopErr)
@@ -12,6 +12,7 @@ import (
"syscall"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
@@ -307,6 +308,8 @@ func (m *Manager) RestoreTier2Whole(stackName string) (WholeRestoreResult, error
// R-1178: the unit half's refusals are asked HERE, before the file half — measured before: files merged, then the
// unit refused (a cut-off database copy) and the app was left STOPPED under a page that said „untouched".
unitDir := tier2UnitDir(destBase)
// D48: the images the unit half pulls in its check stay on the box through the file half and the unit half.
defer dockerexec.HoldImageWork()()
if _, err := m.RestoreFromRecoveryUnitAtWith(stackName, unitDir, UnitRestoreOptions{AcceptMissingFiles: true, CheckOnly: true}); err != nil {
m.logger.Printf("[WARN] [backup] whole restore REFUSED for %s before the file half: the unit half would refuse (%v) — nothing moved", stackName, err)
return res, err