D48: a restore downloads first — every restore path resolves and pulls every image of the definition that will run before the stop, under the image-work lock held until the restore returns; a failed pull refuses with the app untouched (a deleted version and a busy registry read differently), one operator line. Red-proved. Commit only.
gates / gates (push) Successful in 1m26s
gates / gates (push) Successful in 1m26s
This commit is contained in:
@@ -243,6 +243,11 @@ type Manager struct {
|
||||
restorePutBackNotify func(stack string, replayErr error)
|
||||
// restoreNotStartedNotify (R-1270) is called once when a restore's data is back but its app did not start in time.
|
||||
restoreNotStartedNotify func(stack, kind string, cause error)
|
||||
// D48: the restore's image probe and pull (nil → docker; tests set them, TestMain makes every image present), and
|
||||
// the operator notice of a restore refused because an image could not be downloaded.
|
||||
imagePresentFn func(ref string) bool
|
||||
imagePullFn func(ctx context.Context, ref string) (string, error)
|
||||
restorePullRefusedNotify func(stack, ref string, cause error)
|
||||
|
||||
// R-403 unit-REHYDRATE seam — the copy that refills the primary unit from the Tier-2 mirror after
|
||||
// a restore, overridable so the "only when hollow" condition and the byte-identical non-effect are
|
||||
|
||||
@@ -19,6 +19,9 @@ func TestMain(m *testing.M) {
|
||||
healthSettle = 0
|
||||
healthInterval = 2 * time.Millisecond
|
||||
healthTimeout = 50 * time.Millisecond
|
||||
// D48 / R-650: no test in this package reaches Docker for the restore's pre-pull — every image is present unless a
|
||||
// test says otherwise (m.imagePresentFn / m.imagePullFn).
|
||||
imagePresentDefault = func(string) bool { return true }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
|
||||
@@ -946,6 +946,20 @@ func (m *Manager) ReconstituteFromOffsite(ctx context.Context, stack string, ack
|
||||
}
|
||||
}
|
||||
|
||||
// D48: DOWNLOAD FIRST — every image of the definition that will run, before the stop, or a refusal with the app
|
||||
// untouched. Held until this restore returns (R-863).
|
||||
runCompose := filepath.Join(scratchCompose, "docker-compose.yml")
|
||||
if !defineFromSnapshot {
|
||||
if live, ok := m.stackProvider.GetStackComposePath(stack); ok && live != "" {
|
||||
runCompose = live
|
||||
}
|
||||
}
|
||||
releaseImages, pErr := m.prepareRestoreImages(stack, runCompose)
|
||||
if pErr != nil {
|
||||
return res, pErr
|
||||
}
|
||||
defer releaseImages()
|
||||
|
||||
// --- FILES ----------------------------------------------------------------------------------
|
||||
// R-166: mark the stop→restore→start window BEFORE stopping. A controller killed anywhere inside
|
||||
// it used to leave the app down with nothing on disk recording that it was owed a restart — and a
|
||||
|
||||
@@ -79,6 +79,15 @@ func (m *Manager) RestoreApp(stackName, snapshotID string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// D48: the live definition's images are on the box before the stop, or the restore is refused untouched.
|
||||
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok && composePath != "" {
|
||||
releaseImages, pErr := m.prepareRestoreImages(stackName, composePath)
|
||||
if pErr != nil {
|
||||
return pErr
|
||||
}
|
||||
defer releaseImages()
|
||||
}
|
||||
|
||||
// Stop the app before restore
|
||||
if m.isDebug() {
|
||||
m.logger.Printf("[DEBUG] RestoreApp: step 1/4 — stopping app %s", stackName)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
// ── D48 (operator ruling 2026-10-25 08:38; `09` §3 decision 257): a restore downloads first ─────────────────────────
|
||||
//
|
||||
// BEFORE: every restore stopped the app and moved its data first; the first pull was inside the database-only start
|
||||
// (`docker compose up -d <db>`), after the stop. A version that could not be downloaded then ended in the D22 put-back —
|
||||
// the whole stop, copy and pour-back for nothing, and an app that was down the while.
|
||||
//
|
||||
// THE RULE. After the existing refusals and before the first stop or change, every image the definition that will RUN
|
||||
// names is resolved: present on the box, or pulled now. Under the image-work lock held SHARED from the first check
|
||||
// until the restore's last start (R-863: the image clean-up cannot delete a pulled image before the start — its pass
|
||||
// takes the lock exclusively with TryLock and simply does not run meanwhile). A failed pull is a REFUSAL: the app is
|
||||
// not stopped and its data is not moved. The household reads which version could not be downloaded, and why — a
|
||||
// version its maker deleted (Docker answers „manifest unknown", „not found", „pull access denied") reads differently
|
||||
// from a registry that is busy or unreachable (anything else, „toomanyrequests" and timeouts included).
|
||||
|
||||
// ErrRestoreImageUnavailable is the KIND of the refusal: an image of the backup's version could not be downloaded.
|
||||
var ErrRestoreImageUnavailable = errors.New("an image of the backup's version cannot be downloaded")
|
||||
|
||||
// restorePullTimeout bounds one image pull of a restore.
|
||||
const restorePullTimeout = 15 * time.Minute
|
||||
|
||||
// imagePresentDefault / imagePullDefault are the production probe and pull (tests replace them in TestMain, so no test
|
||||
// in this package reaches Docker — R-650; a test that needs a missing image sets m.imagePresentFn / m.imagePullFn).
|
||||
var (
|
||||
imagePresentDefault = func(ref string) bool {
|
||||
return dockerexec.Command("docker", "image", "inspect", "--format", "{{.Id}}", ref).Run() == nil
|
||||
}
|
||||
imagePullDefault = func(ctx context.Context, ref string) (string, error) {
|
||||
out, err := dockerexec.CommandContext(ctx, "docker", "pull", ref).CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
)
|
||||
|
||||
func (m *Manager) imagePresent(ref string) bool {
|
||||
if m.imagePresentFn != nil {
|
||||
return m.imagePresentFn(ref)
|
||||
}
|
||||
return imagePresentDefault(ref)
|
||||
}
|
||||
|
||||
func (m *Manager) imagePull(ctx context.Context, ref string) (string, error) {
|
||||
if m.imagePullFn != nil {
|
||||
return m.imagePullFn(ctx, ref)
|
||||
}
|
||||
return imagePullDefault(ctx, ref)
|
||||
}
|
||||
|
||||
// pullGone reports whether Docker's answer says the version does not exist (deleted by its maker, a wrong name) rather
|
||||
// than that the registry could not be reached or is busy.
|
||||
func pullGone(out string) bool {
|
||||
low := strings.ToLower(out)
|
||||
for _, s := range []string{"manifest unknown", "not found", "repository does not exist", "pull access denied", "name unknown", "no such manifest"} {
|
||||
if strings.Contains(low, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// prepareRestoreImages resolves every image composePath names and pulls the absent ones, holding the image-work lock
|
||||
// until release(). err = the refusal (the household's sentence); the lock is released already then.
|
||||
func (m *Manager) prepareRestoreImages(stack, composePath string) (release func(), err error) {
|
||||
refs := ParseComposeImages(composePath)
|
||||
end := dockerexec.HoldImageWork()
|
||||
for _, ref := range refs {
|
||||
if m.imagePresent(ref) {
|
||||
continue
|
||||
}
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), restorePullTimeout)
|
||||
out, perr := m.imagePull(ctx, ref)
|
||||
cancel()
|
||||
if perr != nil {
|
||||
end()
|
||||
tail := strings.TrimSpace(out)
|
||||
if len(tail) > 300 {
|
||||
tail = tail[len(tail)-300:]
|
||||
}
|
||||
key := "err.backup.restore_image_unreachable"
|
||||
why := "the registry did not answer or is busy"
|
||||
if pullGone(out) {
|
||||
key, why = "err.backup.restore_image_gone", "the version does not exist any more"
|
||||
}
|
||||
m.logger.Printf("[ERROR] [backup] Restore REFUSED for %s before anything was touched: the image %s could not be downloaded (%s; after %s): %v — %s (D48)",
|
||||
stack, ref, why, time.Since(start).Round(time.Second), perr, tail)
|
||||
if m.restorePullRefusedNotify != nil {
|
||||
m.restorePullRefusedNotify(stack, ref, fmt.Errorf("%s: %v", why, perr))
|
||||
}
|
||||
return func() {}, util.MsgErrorf(ErrRestoreImageUnavailable, key, PinsVersion([]string{ref}))
|
||||
}
|
||||
m.logger.Printf("[INFO] [backup] Restore %s: downloaded %s before anything was touched (%s, D48)", stack, ref, time.Since(start).Round(time.Second))
|
||||
}
|
||||
return end, nil
|
||||
}
|
||||
|
||||
// SetRestorePullRefusedNotify wires the operator notice for a restore refused because an image could not be downloaded.
|
||||
func (m *Manager) SetRestorePullRefusedNotify(fn func(stack, ref string, cause error)) {
|
||||
m.restorePullRefusedNotify = fn
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
// D48 (operator ruling 2026-10-25; `09` §3 decision 257) — a restore downloads first. Before (`e1613ae`): the first pull
|
||||
// was inside the database-only start, AFTER the stop; a version that could not be downloaded ended in the D22 put-back.
|
||||
// COMPANION RED-PROOF (REPORT.md): with prepareRestoreImages not called, TestD48_AnUnpullablePinIsRefusedBeforeTheStop
|
||||
// and TestD48_TheOffsiteRestoreToo fail — the app is stopped (the calls start with "stop").
|
||||
|
||||
func d48Unpullable(m *Manager, answer string) *[]string {
|
||||
var pulled []string
|
||||
m.imagePresentFn = func(string) bool { return false }
|
||||
m.imagePullFn = func(_ context.Context, ref string) (string, error) {
|
||||
pulled = append(pulled, ref)
|
||||
return answer, errors.New("exit status 1")
|
||||
}
|
||||
return &pulled
|
||||
}
|
||||
|
||||
// An unpullable pin: refused, the stop never called, nothing poured, no copy made, no definition written.
|
||||
func TestD48_AnUnpullablePinIsRefusedBeforeTheStop(t *testing.T) {
|
||||
m, p, rig, imported := r1177Fixture(t)
|
||||
pulled := d48Unpullable(m, "Error response from daemon: manifest for example/app:9.9.9 not found: manifest unknown")
|
||||
var notified []string
|
||||
m.SetRestorePullRefusedNotify(func(stack, ref string, _ error) { notified = append(notified, stack+" "+ref) })
|
||||
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
if !errors.Is(err, ErrRestoreImageUnavailable) || len(*pulled) == 0 {
|
||||
t.Fatalf("an unpullable pin must refuse, got %v (pulled %v)", err, *pulled)
|
||||
}
|
||||
if want := util.MsgError("err.backup.restore_image_gone", PinsVersion([]string{(*pulled)[0]})).Error(); err.Error() != want {
|
||||
t.Fatalf("a deleted version reads its own sentence, got %q", err.Error())
|
||||
}
|
||||
if p.stopped || len(p.calls) != 0 {
|
||||
t.Fatalf("the app must not be stopped or touched at all, calls %v", p.calls)
|
||||
}
|
||||
if len(rig.poured) != 0 || len(rig.defsWrote) != 0 || len(*imported) != 0 || len(undoCopies(t, rig.undoRoot)) != 0 {
|
||||
t.Fatalf("nothing may move: poured %v, defs %v, imported %v", rig.poured, rig.defsWrote, *imported)
|
||||
}
|
||||
if len(notified) != 1 {
|
||||
t.Fatalf("the operator gets one notice, got %v", notified)
|
||||
}
|
||||
}
|
||||
|
||||
// A registry that is busy or unreachable reads differently from a deleted version.
|
||||
func TestD48_AnUnreachableRegistryReadsDifferently(t *testing.T) {
|
||||
m, p, _, _ := r1177Fixture(t)
|
||||
pulled := d48Unpullable(m, "Error response from daemon: Get \"https://registry-1.docker.io/v2/\": dial tcp: lookup registry-1.docker.io: i/o timeout")
|
||||
_, err := m.RestoreFromRecoveryUnit("app")
|
||||
if len(*pulled) == 0 {
|
||||
t.Fatalf("the restore never asked for the image (err %v)", err)
|
||||
}
|
||||
if want := util.MsgError("err.backup.restore_image_unreachable", PinsVersion([]string{(*pulled)[0]})).Error(); err == nil || err.Error() != want {
|
||||
t.Fatalf("an unreachable registry reads „try again later\", got %v", err)
|
||||
}
|
||||
if p.stopped {
|
||||
t.Fatal("the app must not be stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// A pullable pin: pulled BEFORE the stop, and the restore runs as before.
|
||||
func TestD48_APullablePinIsPulledFirstAndTheRestoreRuns(t *testing.T) {
|
||||
m, p, _, imported := r1177Fixture(t)
|
||||
m.imagePresentFn = func(string) bool { return false }
|
||||
m.imagePullFn = func(_ context.Context, ref string) (string, error) {
|
||||
p.calls = append(p.calls, "pull:"+ref)
|
||||
return "Status: Downloaded newer image", nil
|
||||
}
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatalf("a pullable restore runs: %v", err)
|
||||
}
|
||||
if len(p.calls) == 0 || !strings.HasPrefix(p.calls[0], "pull:") {
|
||||
t.Fatalf("the pull must come before the stop, calls %v", p.calls)
|
||||
}
|
||||
stop := -1
|
||||
for i, c := range p.calls {
|
||||
if c == "stop" {
|
||||
stop = i
|
||||
break
|
||||
}
|
||||
}
|
||||
for i, c := range p.calls {
|
||||
if strings.HasPrefix(c, "pull:") && i > stop {
|
||||
t.Fatalf("every pull before the stop, calls %v", p.calls)
|
||||
}
|
||||
}
|
||||
if len(*imported) != 1 {
|
||||
t.Fatalf("the restore ran as before (one database replayed), got %v", *imported)
|
||||
}
|
||||
}
|
||||
|
||||
// The second drive's whole restore asks the unit half BEFORE its file half (R-1178): the pull is in that check.
|
||||
func TestD48_TheCheckOnlyPassCarriesThePull(t *testing.T) {
|
||||
m, p, _, _ := r1177Fixture(t)
|
||||
d48Unpullable(m, "manifest unknown")
|
||||
unit := m.primaryUnitDirFor("app")
|
||||
if _, err := m.RestoreFromRecoveryUnitAtWith("app", unit, UnitRestoreOptions{CheckOnly: true}); !errors.Is(err, ErrRestoreImageUnavailable) {
|
||||
t.Fatalf("the check must refuse an unpullable pin, got %v", err)
|
||||
}
|
||||
if p.stopped {
|
||||
t.Fatal("the check never stops the app")
|
||||
}
|
||||
}
|
||||
|
||||
// The off-site restore: refused before the stop too.
|
||||
func TestD48_TheOffsiteRestoreToo(t *testing.T) {
|
||||
m, prov, imported := reconFixture(t, "20260719T060000Z", "2026-07-19T06:00:00Z", pgDump(1))
|
||||
d48Unpullable(m, "pull access denied for example/immich, repository does not exist")
|
||||
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
||||
if !errors.Is(err, ErrRestoreImageUnavailable) {
|
||||
t.Fatalf("an off-site restore with an unpullable image must refuse, got %v", err)
|
||||
}
|
||||
if len(prov.calls) != 0 || len(*imported) != 0 {
|
||||
t.Fatalf("the app must not be stopped and nothing replayed: calls %v, imported %v", prov.calls, *imported)
|
||||
}
|
||||
}
|
||||
|
||||
// Docker's answers, as the two sentences read them.
|
||||
func TestD48_PullGone(t *testing.T) {
|
||||
for out, want := range map[string]bool{
|
||||
"manifest for x:1 not found: manifest unknown": true,
|
||||
"pull access denied for x, repository does not exist or may require": true,
|
||||
"toomanyrequests: You have reached your pull rate limit": false,
|
||||
"dial tcp: lookup registry-1.docker.io: i/o timeout": false,
|
||||
"net/http: TLS handshake timeout": false,
|
||||
} {
|
||||
if got := pullGone(out); got != want {
|
||||
t.Errorf("pullGone(%q) = %v, want %v", out, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -463,6 +463,21 @@ func (m *Manager) RestoreFromRecoveryUnitAtWith(stackName, unitDir string, opt U
|
||||
}
|
||||
undoExists = ex
|
||||
}
|
||||
// D48 (`09` §3 decision 257): DOWNLOAD FIRST. Every image the definition that will run names is on the box before
|
||||
// the stop — or the restore is refused here, with the app running and its data where it was. The lock is held
|
||||
// until this function returns, past the last start (R-863). The CheckOnly pass carries it too (R-1178), so the
|
||||
// second drive's file half never runs before a pull that would refuse.
|
||||
runCompose := filepath.Join(composeDir, "docker-compose.yml")
|
||||
if !fileExists(runCompose) {
|
||||
if live, ok := m.stackProvider.GetStackComposePath(stackName); ok && live != "" {
|
||||
runCompose = live
|
||||
}
|
||||
}
|
||||
releaseImages, pErr := m.prepareRestoreImages(stackName, runCompose)
|
||||
if pErr != nil {
|
||||
return res, pErr
|
||||
}
|
||||
defer releaseImages()
|
||||
if opt.CheckOnly {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -420,6 +420,15 @@ func (m *Manager) RestoreTier2Files(stackName string) (filesRestored int, err er
|
||||
}
|
||||
m.logger.Printf("[INFO] [backup] Tier-2 file restore for %s: %s (v2) → %s (additive-only)", stackName, destBase, liveNsRoot)
|
||||
|
||||
// D48: the live definition's images are on the box before the stop, or the restore is refused untouched.
|
||||
if composePath, ok := m.stackProvider.GetStackComposePath(stackName); ok && composePath != "" {
|
||||
releaseImages, pErr := m.prepareRestoreImages(stackName, composePath)
|
||||
if pErr != nil {
|
||||
return 0, pErr
|
||||
}
|
||||
defer releaseImages()
|
||||
}
|
||||
|
||||
// Stop → copy → start → health (the standard restore shape; F17: errors surface, never swallowed).
|
||||
if stopErr := m.stackProvider.StopStack(stackName); stopErr != nil {
|
||||
m.logger.Printf("[WARN] [backup] could not stop %s before Tier-2 file restore: %v (continuing)", stackName, stopErr)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
@@ -307,6 +308,8 @@ func (m *Manager) RestoreTier2Whole(stackName string) (WholeRestoreResult, error
|
||||
// R-1178: the unit half's refusals are asked HERE, before the file half — measured before: files merged, then the
|
||||
// unit refused (a cut-off database copy) and the app was left STOPPED under a page that said „untouched".
|
||||
unitDir := tier2UnitDir(destBase)
|
||||
// D48: the images the unit half pulls in its check stay on the box through the file half and the unit half.
|
||||
defer dockerexec.HoldImageWork()()
|
||||
if _, err := m.RestoreFromRecoveryUnitAtWith(stackName, unitDir, UnitRestoreOptions{AcceptMissingFiles: true, CheckOnly: true}); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] whole restore REFUSED for %s before the file half: the unit half would refuse (%v) — nothing moved", stackName, err)
|
||||
return res, err
|
||||
|
||||
Reference in New Issue
Block a user