.fab exclusion scoping: classes in the manual export (Task 4, v0.136.0)
SQ6 over-capture FIXED for classified apps: the userdata root tar is exclude-scoped (keeps only ancestors/descendants of a SELECTED bind relpath — R1-C, the tier2Reconcile keep-rule); no selected userdata bind → no root tar (radarr state-only). New appbackup.ComputeFabBuckets (shared resolveGuardCollapse pipeline; class buckets; guards over ALL classes; no cross-bucket containment). appexport/fabplan.go: computeFabPlan + tarDirectoryExcluding + fabEstimateSplit. ExportRequest gains DeselectOptional/OptInExcluded (both start handlers — two-call-site); mandatory is a server-side floor. Manifest v1 + import UNTOUCHED; legacy apps byte-identical to v0.130.0. Estimate additive class split; export UI: locked-mandatory/optional-checkboxes/excluded-opt-in + two-number warning. All 6 §10 red-proofs verified. 6D Accept #1 now runs against this shape.
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// fabWebProvider is a minimal appexport.ExportStackProvider: a config+userdata app (no volumes, no DB)
|
||||
// so a real export runs without docker, and classified so the plan applies.
|
||||
type fabWebProvider struct {
|
||||
stackDir, stacksDir, hddPath string
|
||||
binds []appbackup.ClassifiedBind
|
||||
}
|
||||
|
||||
func (p *fabWebProvider) GetStackDir(string) (string, bool) { return p.stackDir, true }
|
||||
func (p *fabWebProvider) GetStackComposePath(string) (string, bool) {
|
||||
return filepath.Join(p.stackDir, "docker-compose.yml"), true
|
||||
}
|
||||
func (p *fabWebProvider) GetStackHDDMounts(string) []string { return []string{appbackup.UserdataDir(p.hddPath)} }
|
||||
func (p *fabWebProvider) GetStackHDDPath(string) string { return p.hddPath }
|
||||
func (p *fabWebProvider) GetStackClassifiedBinds(string) ([]appbackup.ClassifiedBind, bool) {
|
||||
return p.binds, true
|
||||
}
|
||||
func (p *fabWebProvider) IsStackRunning(string) bool { return false }
|
||||
func (p *fabWebProvider) StopStack(string) error { return nil }
|
||||
func (p *fabWebProvider) StartStack(string) error { return nil }
|
||||
func (p *fabWebProvider) GetStackDisplayName(n string) string { return n }
|
||||
func (p *fabWebProvider) GetStackNeedsHDD(string) bool { return true }
|
||||
func (p *fabWebProvider) GetDockerVolumes(string) []string { return nil }
|
||||
func (p *fabWebProvider) IsStackDeployed(string) bool { return true }
|
||||
func (p *fabWebProvider) GetDecryptedEnv(string) map[string]string { return nil }
|
||||
func (p *fabWebProvider) GetStacksBaseDir() string { return p.stacksDir }
|
||||
func (p *fabWebProvider) SaveEncryptedAppConfig(string, map[string]string) error { return nil }
|
||||
func (p *fabWebProvider) RefreshStacks() error { return nil }
|
||||
func (p *fabWebProvider) RemoveStackVolumes(string) error { return nil }
|
||||
|
||||
// waitExportDone polls the exporter until the active job finishes.
|
||||
func waitExportDone(t *testing.T, e *appexport.Exporter) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if j := e.GetActiveJob(); j != nil {
|
||||
if done, _ := j.Snapshot()["done"].(bool); done {
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("export did not finish")
|
||||
}
|
||||
|
||||
// findFab returns the newest .fab under dir.
|
||||
func findFab(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
var found string
|
||||
filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && strings.HasSuffix(p, ".fab") {
|
||||
found = p
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if found == "" {
|
||||
t.Fatalf("no .fab under %s", dir)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// userdataTarEntries extracts data/hdd/userdata.tar from an UNENCRYPTED .fab and lists its entries.
|
||||
func userdataTarEntries(t *testing.T, fabPath string) []string {
|
||||
t.Helper()
|
||||
f, err := os.Open(fabPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
var inner []byte
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.HasSuffix(filepath.ToSlash(h.Name), "hdd/userdata.tar") {
|
||||
inner, _ = io.ReadAll(tr)
|
||||
}
|
||||
}
|
||||
if inner == nil {
|
||||
return nil // no userdata tar in the bundle
|
||||
}
|
||||
itr := tar.NewReader(strings.NewReader(string(inner)))
|
||||
var names []string
|
||||
for {
|
||||
h, err := itr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = append(names, filepath.ToSlash(h.Name))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// §7-F / red-proof F: BOTH start pipelines must carry the class selection. Each produces a bundle;
|
||||
// with OptInExcluded, the excluded content must be present in the userdata tar. Red-proof: drop the
|
||||
// selection fields from one start handler → that pipeline's assertion fails (two-call-site).
|
||||
func TestFab_SelectionsRideBothStartPipelines(t *testing.T) {
|
||||
build := func(t *testing.T) (*Server, *fabWebProvider, *appexport.Exporter, string) {
|
||||
s := testServer(t)
|
||||
s.cfg.Paths.DataDir = t.TempDir()
|
||||
drive := t.TempDir()
|
||||
stackDir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(stackDir, "docker-compose.yml"), []byte("services: {}\n"), 0644)
|
||||
fabWrite(t, drive, "userdata/media/books/a.epub", "BOOK")
|
||||
fabWrite(t, drive, "userdata/media/movies/big.mkv", "MOVIE") // excluded, opted-in below
|
||||
prov := &fabWebProvider{stackDir: stackDir, stacksDir: t.TempDir(), hddPath: drive,
|
||||
binds: []appbackup.ClassifiedBind{
|
||||
{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootUserdata, RelPath: "media/books"}, Class: appbackup.ClassMandatory},
|
||||
{ComposeBind: appbackup.ComposeBind{Root: appbackup.RootUserdata, RelPath: "media/movies"}, Class: appbackup.ClassExcluded},
|
||||
}}
|
||||
e := appexport.NewExporter(prov, s.logger, "test")
|
||||
s.appExporter = e
|
||||
if err := s.settings.AddStoragePath(settings.StoragePath{Path: drive, Label: "d", Schedulable: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, prov, e, drive
|
||||
}
|
||||
|
||||
post := func(s *Server, path, body string, h func(http.ResponseWriter, *http.Request)) {
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
h(rr, req)
|
||||
var resp map[string]interface{}
|
||||
json.Unmarshal(rr.Body.Bytes(), &resp)
|
||||
if resp["ok"] != true {
|
||||
// jsonResponse uses {"ok":true}; some paths wrap in {"data":..}. Accept 200.
|
||||
if rr.Code != http.StatusOK {
|
||||
panic("start failed: " + rr.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pipeline 1: /api/export/start (dest = the registered drive).
|
||||
t.Run("api/export/start", func(t *testing.T) {
|
||||
s, _, e, drive := build(t)
|
||||
body, _ := json.Marshal(map[string]interface{}{"stack_name": "calibre-web", "dest_drive": drive, "opt_in_excluded": []string{"userdata/media/movies"}})
|
||||
post(s, "/api/export/start", string(body), s.apiExportStart)
|
||||
waitExportDone(t, e)
|
||||
got := userdataTarEntries(t, findFab(t, filepath.Join(drive, "exports")))
|
||||
if !hasSuffixIn(got, "media/movies/big.mkv") {
|
||||
t.Errorf("start pipeline dropped opt_in_excluded — movies absent: %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
// Pipeline 2: /api/export/download/start (dest = the staging download dir).
|
||||
t.Run("api/export/download/start", func(t *testing.T) {
|
||||
s, _, e, _ := build(t)
|
||||
body := `{"stack_name":"calibre-web","opt_in_excluded":["userdata/media/movies"]}`
|
||||
post(s, "/api/export/download/start", body, s.apiExportDownloadStart)
|
||||
waitExportDone(t, e)
|
||||
got := userdataTarEntries(t, findFab(t, s.fabDownloadDir()))
|
||||
if !hasSuffixIn(got, "media/movies/big.mkv") {
|
||||
t.Errorf("download pipeline dropped opt_in_excluded — movies absent: %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func fabWrite(t *testing.T, root, rel, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func hasSuffixIn(ss []string, suffix string) bool {
|
||||
for _, s := range ss {
|
||||
if strings.HasSuffix(filepath.ToSlash(s), suffix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -168,10 +168,12 @@ func (s *Server) apiExportStart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
var req struct {
|
||||
StackName string `json:"stack_name"`
|
||||
DestDrive string `json:"dest_drive"`
|
||||
Password string `json:"password"`
|
||||
StopApp bool `json:"stop_app"`
|
||||
StackName string `json:"stack_name"`
|
||||
DestDrive string `json:"dest_drive"`
|
||||
Password string `json:"password"`
|
||||
StopApp bool `json:"stop_app"`
|
||||
DeselectOptional []string `json:"deselect_optional"` // Task 4: `.fab` class selection
|
||||
OptInExcluded []string `json:"opt_in_excluded"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
s.logger.Printf("[DEBUG] [web] apiExportStart: invalid body: %v", err)
|
||||
@@ -199,10 +201,12 @@ func (s *Server) apiExportStart(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
err := s.appExporter.StartExport(appexport.ExportRequest{
|
||||
StackName: req.StackName,
|
||||
DestDrive: req.DestDrive,
|
||||
Password: req.Password,
|
||||
StopApp: req.StopApp,
|
||||
StackName: req.StackName,
|
||||
DestDrive: req.DestDrive,
|
||||
Password: req.Password,
|
||||
StopApp: req.StopApp,
|
||||
DeselectOptional: req.DeselectOptional,
|
||||
OptInExcluded: req.OptInExcluded,
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] [web] Export start failed for %s: %v", req.StackName, err)
|
||||
|
||||
@@ -78,9 +78,11 @@ func (s *Server) apiExportDownloadStart(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
StackName string `json:"stack_name"`
|
||||
Password string `json:"password"`
|
||||
StopApp bool `json:"stop_app"`
|
||||
StackName string `json:"stack_name"`
|
||||
Password string `json:"password"`
|
||||
StopApp bool `json:"stop_app"`
|
||||
DeselectOptional []string `json:"deselect_optional"` // Task 4: `.fab` class selection (two-call-site)
|
||||
OptInExcluded []string `json:"opt_in_excluded"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
jsonError(w, "Invalid request body", http.StatusBadRequest)
|
||||
@@ -98,10 +100,12 @@ func (s *Server) apiExportDownloadStart(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
// A concurrent export/import gets the exporter's own busy answer (single-flight).
|
||||
if err := s.appExporter.StartExport(appexport.ExportRequest{
|
||||
StackName: req.StackName,
|
||||
DestDrive: s.fabDownloadRoot(),
|
||||
Password: req.Password,
|
||||
StopApp: req.StopApp,
|
||||
StackName: req.StackName,
|
||||
DestDrive: s.fabDownloadRoot(),
|
||||
Password: req.Password,
|
||||
StopApp: req.StopApp,
|
||||
DeselectOptional: req.DeselectOptional,
|
||||
OptInExcluded: req.OptInExcluded,
|
||||
}); err != nil {
|
||||
s.logger.Printf("[ERROR] [web] download-export start failed for %s: %v", req.StackName, err)
|
||||
jsonError(w, err.Error(), http.StatusConflict)
|
||||
|
||||
@@ -37,6 +37,23 @@
|
||||
<div id="estWarning" style="display:none;color:var(--crit);margin-top:.5rem;font-weight:600"></div>
|
||||
</div>
|
||||
|
||||
<div id="classSelect" style="display:none;margin-bottom:1.5rem">
|
||||
<div id="mandBox" style="display:none;margin-bottom:1rem">
|
||||
<div style="font-weight:600;margin-bottom:.25rem">Mindig része a mentésnek:</div>
|
||||
<div id="mandList" style="color:var(--text-3);font-size:.9rem"></div>
|
||||
</div>
|
||||
<div id="optBox" style="display:none;margin-bottom:1rem">
|
||||
<div style="font-weight:600;margin-bottom:.25rem">Választható tartalom:</div>
|
||||
<div id="optList"></div>
|
||||
</div>
|
||||
<div id="exclBox" style="display:none">
|
||||
<div id="exclSummary" style="margin-bottom:.25rem"></div>
|
||||
<div style="color:var(--text-3);font-size:.85rem;margin-bottom:.5rem">A kihagyott mappák tartalma a Fájlkezelőben bármikor elérhető, és külön is lementhető.</div>
|
||||
<button type="button" class="btn btn-xs btn-outline" onclick="toggleExcl()" id="exclToggle">Kihagyott tartalom megjelenítése</button>
|
||||
<div id="exclList" style="display:none;margin-top:.5rem"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3>Jelszó (opcionális)</h3>
|
||||
<div style="display:flex;gap:.5rem;margin-bottom:1rem">
|
||||
<input type="password" id="exportPassword" placeholder="Titkosítási jelszó" style="flex:1;padding:.5rem">
|
||||
@@ -122,11 +139,84 @@ async function loadEstimate() {
|
||||
btn.disabled = false;
|
||||
}
|
||||
box.style.display = 'block';
|
||||
renderClassSelect(est);
|
||||
} catch(e) {
|
||||
console.error('Estimate error:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Task 4: the class-scoped selection UI (classified apps only; legacy apps see the plain estimate).
|
||||
var lastEst = null;
|
||||
function humanBytes(b) {
|
||||
if (b >= 1073741824) return (b/1073741824).toFixed(1) + ' GB';
|
||||
if (b >= 1048576) return (b/1048576).toFixed(1) + ' MB';
|
||||
if (b >= 1024) return (b/1024).toFixed(1) + ' KB';
|
||||
return b + ' B';
|
||||
}
|
||||
function renderClassSelect(est) {
|
||||
lastEst = est;
|
||||
var wrap = document.getElementById('classSelect');
|
||||
if (!est.has_classification) { wrap.style.display = 'none'; return; }
|
||||
wrap.style.display = 'block';
|
||||
|
||||
var mand = est.mandatory_items || [], opt = est.optional_items || [], excl = est.excluded_items || [];
|
||||
// Mandatory (locked).
|
||||
var mb = document.getElementById('mandBox');
|
||||
if (mand.length) {
|
||||
mb.style.display = 'block';
|
||||
document.getElementById('mandList').innerHTML = mand.map(function(i){
|
||||
return '<div>' + esc(i.rel_path) + ' <span style="opacity:.7">(' + i.human + ')</span></div>';
|
||||
}).join('');
|
||||
} else { mb.style.display = 'none'; }
|
||||
// Optional (pre-selected checkboxes).
|
||||
var ob = document.getElementById('optBox');
|
||||
if (opt.length) {
|
||||
ob.style.display = 'block';
|
||||
document.getElementById('optList').innerHTML = opt.map(function(i){
|
||||
return '<label style="display:flex;gap:.5rem;align-items:center;cursor:pointer;padding:.15rem 0">' +
|
||||
'<input type="checkbox" class="fab-opt" checked data-key="' + esc(i.key) + '" data-bytes="' + i.bytes + '" onchange="recomputeTotal()">' +
|
||||
'<span>' + esc(i.rel_path) + ' <span style="opacity:.7">(' + i.human + ')</span></span></label>';
|
||||
}).join('');
|
||||
} else { ob.style.display = 'none'; }
|
||||
// Excluded (opt-in, collapsed, behind the two-number warning).
|
||||
var eb = document.getElementById('exclBox');
|
||||
if (excl.length) {
|
||||
eb.style.display = 'block';
|
||||
document.getElementById('exclList').innerHTML = excl.map(function(i){
|
||||
return '<label style="display:flex;gap:.5rem;align-items:center;cursor:pointer;padding:.15rem 0">' +
|
||||
'<input type="checkbox" class="fab-excl" data-key="' + esc(i.key) + '" data-bytes="' + i.bytes + '" onchange="recomputeTotal()">' +
|
||||
'<span>' + esc(i.rel_path) + ' <span style="opacity:.7">(' + i.human + ')</span></span></label>';
|
||||
}).join('');
|
||||
} else { eb.style.display = 'none'; }
|
||||
recomputeTotal();
|
||||
}
|
||||
function recomputeTotal() {
|
||||
if (!lastEst) return;
|
||||
var base = lastEst.base_bytes || 0;
|
||||
var optSum = 0, exclSum = 0, exclAll = 0;
|
||||
document.querySelectorAll('.fab-opt').forEach(function(cb){ if (cb.checked) optSum += parseInt(cb.dataset.bytes,10)||0; });
|
||||
document.querySelectorAll('.fab-excl').forEach(function(cb){ var b = parseInt(cb.dataset.bytes,10)||0; exclAll += b; if (cb.checked) exclSum += b; });
|
||||
var alap = base + optSum;
|
||||
var full = alap + exclAll;
|
||||
var s = document.getElementById('exclSummary');
|
||||
if ((lastEst.excluded_items||[]).length) {
|
||||
s.innerHTML = 'Alap mentés: ~' + humanBytes(alap) + '. A kihagyott, nagy méretű tartalommal együtt: ~' + humanBytes(full) + '.';
|
||||
}
|
||||
}
|
||||
function toggleExcl() {
|
||||
var l = document.getElementById('exclList');
|
||||
var shown = l.style.display !== 'none';
|
||||
l.style.display = shown ? 'none' : 'block';
|
||||
document.getElementById('exclToggle').textContent = shown ? 'Kihagyott tartalom megjelenítése' : 'Kihagyott tartalom elrejtése';
|
||||
}
|
||||
function gatherSelections() {
|
||||
var deselect = [], optIn = [];
|
||||
document.querySelectorAll('.fab-opt').forEach(function(cb){ if (!cb.checked) deselect.push(cb.dataset.key); });
|
||||
document.querySelectorAll('.fab-excl').forEach(function(cb){ if (cb.checked) optIn.push(cb.dataset.key); });
|
||||
return { deselect_optional: deselect, opt_in_excluded: optIn };
|
||||
}
|
||||
function esc(s) { return String(s).replace(/[&<>"]/g, function(c){ return {'&':'&','<':'<','>':'>','"':'"'}[c]; }); }
|
||||
|
||||
async function startExport() {
|
||||
var drive = document.getElementById('destDrive').value;
|
||||
var password = document.getElementById('exportPassword').value;
|
||||
@@ -137,6 +227,7 @@ async function startExport() {
|
||||
document.getElementById('doneCard').style.display = 'none';
|
||||
|
||||
try {
|
||||
var sel = gatherSelections();
|
||||
var resp = await fetch('/api/export/start', {
|
||||
method: 'POST',
|
||||
headers: csrfH(),
|
||||
@@ -144,7 +235,9 @@ async function startExport() {
|
||||
stack_name: stackName,
|
||||
dest_drive: drive,
|
||||
password: password,
|
||||
stop_app: stopApp
|
||||
stop_app: stopApp,
|
||||
deselect_optional: sel.deselect_optional,
|
||||
opt_in_excluded: sel.opt_in_excluded
|
||||
})
|
||||
});
|
||||
var data = await resp.json();
|
||||
|
||||
Reference in New Issue
Block a user