R-1256: four restore edges (review round 2 HUNT-1 H4/H5/H6/H8) — each test seen failing first
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
H4: a failed definition write in the off-site restore writes the live definition back before the start;
a failed write-back holds the app (restore_mixed).
H5: a put-back whose stop fails pours nothing, keeps the copy and holds the app (off-site, local, power-cut);
a failed definition write-back keeps the copy (D20); a failed 'volume rm -f' fails that volume (no merge).
H6: TryBeginRestoreOp/RestoreBusy: check every operation and record the restore in one step under m.mu;
every web handler and the kept loads begin through it; install-time kept load asks about the mirror too.
H8: SweepUnusedRestoreCopies every 6 h: removes a copy only with no hold, no marker naming it, no restore
running, and older than 14 days.
Commit only; rides the next controller release.
This commit is contained in:
1 parent
ed3927f03a
commit
342a394bff
20 files changed
+860
-50
No files matched your search
@@ -94,7 +94,9 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: data.UseOff, Data: data})
|
||||
return true
|
||||
}
|
||||
if st := r.backupMgr.RestoreStatus(); st.Running || r.backupMgr.IsRunning() {
|
||||
// R-1256 H6: every operation — a restore, a backup run, a second-drive copy — not only the first two; and the
|
||||
// load itself begins in the same step as this question (startKeptLoad → ErrRestoreBusy).
|
||||
if busy, _ := r.backupMgr.RestoreBusy(); busy != "" {
|
||||
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
|
||||
return true
|
||||
}
|
||||
@@ -104,7 +106,11 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
}
|
||||
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (tier %d, %s) under the kept files %v", name, src, cp.c.Tier, cp.c.Time.UTC().Format(time.RFC3339), old)
|
||||
r.dropLeftoverHold(name, "use-kept-data") // R-704: this is a new install of an app that is not installed
|
||||
r.startKeptLoad(name, hdd, cp.c, lang, nil)
|
||||
if err := r.startKeptLoad(name, hdd, cp.c, lang, nil); err != nil {
|
||||
r.logger.Printf("[WARN] [api] Deploy %s: use my kept data REFUSED at its start: %v", name, err)
|
||||
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
|
||||
return true
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, apiResponse{OK: true, Message: r.msgLang(lang, "kept.load.started", display)})
|
||||
return true
|
||||
default:
|
||||
@@ -120,7 +126,7 @@ func (r *Router) keptCopyName(lang string, c backup.KeptCopy) string {
|
||||
|
||||
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last. The
|
||||
// off-site copy goes through LoadKeptOffsite (download the unit alone, judge it, then the same restore).
|
||||
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) {
|
||||
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) error {
|
||||
display := name
|
||||
if st, ok := r.stackMgr.GetStack(name); ok && st.Meta.DisplayName != "" {
|
||||
display = st.Meta.DisplayName
|
||||
@@ -128,10 +134,9 @@ func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang strin
|
||||
okMsg := func(error) string { return r.msgLang(lang, "kept.load.done", display) }
|
||||
failMsg := func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) }
|
||||
if c.Tier == backup.KeptTierOffsite {
|
||||
r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
return
|
||||
return r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
}
|
||||
r.backupMgr.LoadKeptApp(name, c.UnitDir, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
return r.backupMgr.LoadKeptApp(name, c.UnitDir, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
}
|
||||
|
||||
// keptAfterLoad runs the app's after_load when the load succeeded, then then(ok).
|
||||
|
||||
@@ -162,7 +162,10 @@ func TestR691_InstallChoiceNamesTheOffsiteCopy(t *testing.T) {
|
||||
// copy, restic and the unit restore behind their seams — no Docker).
|
||||
// COMPANION RED-PROOF: remove the dropLeftoverHold call from keptDataAtInstall → the hold is still there
|
||||
// after the 202 and the first assertion fails; remove it from deployStack → the AST check fails.
|
||||
func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
|
||||
// keptInstallRig is an installable app with old data on its drive and an off-site copy of its unit; done receives
|
||||
// once per unit restore the load reaches.
|
||||
func keptInstallRig(t *testing.T) (*Router, *settings.Settings, string, chan struct{}) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
drive := filepath.Join(dir, "drive")
|
||||
cfg := &config.Config{}
|
||||
@@ -234,6 +237,11 @@ func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
|
||||
return backup.UnitRestoreResult{}, nil
|
||||
})
|
||||
r := &Router{stackMgr: m, backupMgr: bm, sett: sett, logger: log.New(io.Discard, "", 0)}
|
||||
return r, sett, drive, done
|
||||
}
|
||||
|
||||
func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
|
||||
r, sett, drive, done := keptInstallRig(t)
|
||||
|
||||
// A leftover update hold from an earlier install (the demo-hp shape): the new install drops it.
|
||||
_ = sett.SetRestoreHold(settings.RestoreHold{Stack: "cloudapp", At: "2026-09-13T19:56:32Z", Reason: settings.HoldReasonUpdateFailed})
|
||||
@@ -287,3 +295,38 @@ func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
|
||||
t.Fatalf("deployStack must call dropLeftoverHold before DeployStack (drop=%v deploy=%v)", dropAt, deployAt)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1256 H6: the install-time „use my kept data" asks about EVERY operation — a second-drive copy too, not only a
|
||||
// restore record and the backup run — and the load begins in the same step as the question.
|
||||
// RED-PROOF: restore the RestoreStatus/IsRunning-only check and LoadKept*'s plain BeginRestoreOp → 202 and the load
|
||||
// runs beside the copy → FAILS.
|
||||
func TestR1256_H6_AKeptLoadAtInstallIsRefusedWhileASecondDriveCopyRuns(t *testing.T) {
|
||||
r, _, drive, done := keptInstallRig(t)
|
||||
end := backup.MarkTier2CopyForTest("someapp")
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang=en", nil)
|
||||
r.keptDataAtInstall(w, req, "cloudapp", drive, "use")
|
||||
end()
|
||||
if w.Code != http.StatusConflict {
|
||||
t.Fatalf("a load during a second-drive copy: code %d, want 409 — body %s", w.Code, w.Body.String())
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("the load ran beside the second-drive copy")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
if st := r.backupMgr.RestoreStatus(); st.Running {
|
||||
t.Fatal("a refused load left a restore op recorded")
|
||||
}
|
||||
// The copy ended: the same press loads.
|
||||
w = httptest.NewRecorder()
|
||||
r.keptDataAtInstall(w, httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang=en", nil), "cloudapp", drive, "use")
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Fatalf("after the copy: code %d — body %s", w.Code, w.Body.String())
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the load never reached the unit restore")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user