R-1256: four restore edges (review round 2 HUNT-1 H4/H5/H6/H8) — each test seen failing first
gates / gates (push) Successful in 31s

H4: a failed definition write in the off-site restore writes the live definition back before the start;
    a failed write-back holds the app (restore_mixed).
H5: a put-back whose stop fails pours nothing, keeps the copy and holds the app (off-site, local, power-cut);
    a failed definition write-back keeps the copy (D20); a failed 'volume rm -f' fails that volume (no merge).
H6: TryBeginRestoreOp/RestoreBusy: check every operation and record the restore in one step under m.mu;
    every web handler and the kept loads begin through it; install-time kept load asks about the mirror too.
H8: SweepUnusedRestoreCopies every 6 h: removes a copy only with no hold, no marker naming it, no restore
    running, and older than 14 days.

Commit only; rides the next controller release.
This commit is contained in:
admin committed 2026-10-28 22:24:16 +01:00
1 parent ed3927f03a
commit 342a394bff
20 files changed
+860 -50

No files matched your search

+11 -6
View File
@@ -94,7 +94,9 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: data.UseOff, Data: data})
return true
}
if st := r.backupMgr.RestoreStatus(); st.Running || r.backupMgr.IsRunning() {
// R-1256 H6: every operation — a restore, a backup run, a second-drive copy — not only the first two; and the
// load itself begins in the same step as this question (startKeptLoad → ErrRestoreBusy).
if busy, _ := r.backupMgr.RestoreBusy(); busy != "" {
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
return true
}
@@ -104,7 +106,11 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
}
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (tier %d, %s) under the kept files %v", name, src, cp.c.Tier, cp.c.Time.UTC().Format(time.RFC3339), old)
r.dropLeftoverHold(name, "use-kept-data") // R-704: this is a new install of an app that is not installed
r.startKeptLoad(name, hdd, cp.c, lang, nil)
if err := r.startKeptLoad(name, hdd, cp.c, lang, nil); err != nil {
r.logger.Printf("[WARN] [api] Deploy %s: use my kept data REFUSED at its start: %v", name, err)
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
return true
}
writeJSON(w, http.StatusAccepted, apiResponse{OK: true, Message: r.msgLang(lang, "kept.load.started", display)})
return true
default:
@@ -120,7 +126,7 @@ func (r *Router) keptCopyName(lang string, c backup.KeptCopy) string {
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last. The
// off-site copy goes through LoadKeptOffsite (download the unit alone, judge it, then the same restore).
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) {
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) error {
display := name
if st, ok := r.stackMgr.GetStack(name); ok && st.Meta.DisplayName != "" {
display = st.Meta.DisplayName
@@ -128,10 +134,9 @@ func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang strin
okMsg := func(error) string { return r.msgLang(lang, "kept.load.done", display) }
failMsg := func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) }
if c.Tier == backup.KeptTierOffsite {
r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
return
return r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
}
r.backupMgr.LoadKeptApp(name, c.UnitDir, okMsg, failMsg, r.keptAfterLoad(name, then))
return r.backupMgr.LoadKeptApp(name, c.UnitDir, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
}
// keptAfterLoad runs the app's after_load when the load succeeded, then then(ok).
+44 -1
View File
@@ -162,7 +162,10 @@ func TestR691_InstallChoiceNamesTheOffsiteCopy(t *testing.T) {
// copy, restic and the unit restore behind their seams — no Docker).
// COMPANION RED-PROOF: remove the dropLeftoverHold call from keptDataAtInstall → the hold is still there
// after the 202 and the first assertion fails; remove it from deployStack → the AST check fails.
func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
// keptInstallRig is an installable app with old data on its drive and an off-site copy of its unit; done receives
// once per unit restore the load reaches.
func keptInstallRig(t *testing.T) (*Router, *settings.Settings, string, chan struct{}) {
t.Helper()
dir := t.TempDir()
drive := filepath.Join(dir, "drive")
cfg := &config.Config{}
@@ -234,6 +237,11 @@ func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
return backup.UnitRestoreResult{}, nil
})
r := &Router{stackMgr: m, backupMgr: bm, sett: sett, logger: log.New(io.Discard, "", 0)}
return r, sett, drive, done
}
func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
r, sett, drive, done := keptInstallRig(t)
// A leftover update hold from an earlier install (the demo-hp shape): the new install drops it.
_ = sett.SetRestoreHold(settings.RestoreHold{Stack: "cloudapp", At: "2026-09-13T19:56:32Z", Reason: settings.HoldReasonUpdateFailed})
@@ -287,3 +295,38 @@ func TestR704_AFreshInstallDropsALeftoverHold(t *testing.T) {
t.Fatalf("deployStack must call dropLeftoverHold before DeployStack (drop=%v deploy=%v)", dropAt, deployAt)
}
}
// R-1256 H6: the install-time „use my kept data" asks about EVERY operation — a second-drive copy too, not only a
// restore record and the backup run — and the load begins in the same step as the question.
// RED-PROOF: restore the RestoreStatus/IsRunning-only check and LoadKept*'s plain BeginRestoreOp → 202 and the load
// runs beside the copy → FAILS.
func TestR1256_H6_AKeptLoadAtInstallIsRefusedWhileASecondDriveCopyRuns(t *testing.T) {
r, _, drive, done := keptInstallRig(t)
end := backup.MarkTier2CopyForTest("someapp")
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang=en", nil)
r.keptDataAtInstall(w, req, "cloudapp", drive, "use")
end()
if w.Code != http.StatusConflict {
t.Fatalf("a load during a second-drive copy: code %d, want 409 — body %s", w.Code, w.Body.String())
}
select {
case <-done:
t.Fatal("the load ran beside the second-drive copy")
case <-time.After(300 * time.Millisecond):
}
if st := r.backupMgr.RestoreStatus(); st.Running {
t.Fatal("a refused load left a restore op recorded")
}
// The copy ended: the same press loads.
w = httptest.NewRecorder()
r.keptDataAtInstall(w, httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang=en", nil), "cloudapp", drive, "use")
if w.Code != http.StatusAccepted {
t.Fatalf("after the copy: code %d — body %s", w.Code, w.Body.String())
}
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("the load never reached the unit restore")
}
}