Files
felhom-controller/controller/internal/web/recovery_test.go
T
admin a3499d1807
gates / gates (push) Successful in 9s
v0.201.0 — a correct recovery code is never called wrong again (CAMPAIGN-11) — MinAgent 0.125.0
R-216: the offsite key recovery is a coupled feature and now says so. featureProbes +
featureMinAgent 0.125.0 + a Supports gate at the unlock entry point, FAILING CLOSED — an
agent that cannot answer is named as such instead of the customer's code being blamed.
Measured live: a 404 from agent 0.120.0 came back as "we did not accept your recovery
code, check that all ten words", in 0.134 s, against a perfect code.

R-218: delete the repo-password short-circuit in needsOffsiteCredential. The declaration
stops when the TIER WORKS, not when a key exists — installing a key is the recovery
screen's whole job, so succeeding at recovery was switching off the mechanism that would
have delivered the coordinates to use it.

R-219: the unlock finishes the job — place the key, bring the tier up, then list. Without
it the promised listing could never render on the shape the screen exists for.

R-217: an unreadable store no longer claims to have opened with unattributable content
(the OffsiteInventory{} zero value). Opened / empty / unreadable are three states.

R-222: a code that is right about a RETAINED earlier package is named, not blamed. States
what the hub knows and promises nothing — no read path exists.

R-215: GET /recovery is gated on the same predicate as the interception.

Five red-proofs, each demonstrated failing and restored.
2026-08-05 17:48:08 +02:00

498 lines
20 KiB
Go

package web
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// R-193 — the recovery screen. HANDLER-LEVEL tests throughout: a test that reaches a helper while the
// mutation lives in the handler cannot observe it, which is how a red-proof passed three sessions ago.
// Everything below drives the real handler (or the real mux) and asserts the rendered page, the
// on-disk effect, or the absence of the code.
const testRecoveryCode = "helyre-allitasi-kod-tiz-szo-pontosan-igy-ni-most"
const testRepoPW = "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1"
// fakeRecoverer is the agent seam. It records every code it was handed so a test can prove the
// handler passed the RIGHT one, and can fail on demand for the wrong-code path.
type fakeRecoverer struct {
mu sync.Mutex
pw string
sha string
fail bool
codes []string
}
func (f *fakeRecoverer) RecoverOffsiteRepoPassword(_ context.Context, code string) (string, string, error) {
f.mu.Lock()
f.codes = append(f.codes, code)
f.mu.Unlock()
if f.fail {
// The shape the agent returns: names the STEP, never the code.
return "", "", fmt.Errorf("unseal failed: age: incorrect passphrase")
}
return f.pw, f.sha, nil
}
// recoveryRunner is the restic seam for the post-unlock inventory.
type recoveryRunner struct {
snapshots []map[string]any
statsSize int64
}
func (rr *recoveryRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, " snapshots"):
b, _ := json.Marshal(rr.snapshots)
return b, nil
case strings.Contains(joined, " stats "):
b, _ := json.Marshal(map[string]any{"total_size": rr.statsSize})
return b, nil
}
return []byte(""), nil
}
type recoveryFixture struct {
s *Server
mgr *backup.Manager
sett *settings.Settings
rec *fakeRecoverer
runner *recoveryRunner
dataDir string
}
// newRecoveryFixture builds a Server in the REBUILT-BOX shape by default: a claimed box (password
// set), no repository password on disk, and the hub holding a sealed package.
func newRecoveryFixture(t *testing.T) *recoveryFixture {
t.Helper()
lg := log.New(io.Discard, "", 0)
dir := t.TempDir()
cfg := &config.Config{}
cfg.Paths.DataDir = filepath.Join(dir, "data")
cfg.Paths.SystemDataPath = filepath.Join(dir, "sys")
cfg.Paths.StacksDir = filepath.Join(dir, "stacks")
cfg.Web.SessionSecret = "test-session-secret-abcdef"
cfg.Web.PasswordHash = "$2a$10$abcdefghijklmnopqrstuv" // claimed: auth is enabled
sett, err := settings.Load(filepath.Join(dir, "settings.json"), lg)
if err != nil {
t.Fatal(err)
}
if err := sett.SetHubEscrowIdentityPresent(true); err != nil { // the hub holds a package
t.Fatal(err)
}
// The realistic shape once the credential self-heal has re-applied the tier: coordinates exist,
// but this box holds no repository password for the history they point at.
if err := sett.SetOffboxTarget(&settings.OffboxTarget{
Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo",
Schedule: "daily", EscrowState: "escrowed",
}); err != nil {
t.Fatal(err)
}
mgr := backup.NewManager(cfg, sett, lg)
if err := mgr.WriteOffboxSecrets("PRIVATE-KEY-MATERIAL", "nas.local ssh-ed25519 AAAAhostkey"); err != nil {
t.Fatal(err)
}
// WriteOffboxSecrets auto-generates a repository password — remove it, because "this box cannot
// open the inherited history" is the whole precondition of the screen.
if err := os.Remove(filepath.Join(cfg.Paths.DataDir, "offbox", "repo_password")); err != nil {
t.Fatal(err)
}
rr := &recoveryRunner{statsSize: 4 << 20}
mgr.SetOffboxRunner(rr.run)
rec := &fakeRecoverer{pw: testRepoPW, sha: backup.HashResticPassword(testRepoPW)}
stackMgr, serr := stacks.NewManager(cfg, lg)
if serr != nil {
t.Fatal(serr)
}
s := &Server{cfg: cfg, settings: sett, backupMgr: mgr, stackMgr: stackMgr, logger: lg, version: "test"}
s.loadTemplates()
s.SetRecoveryRecoverer(func() (backup.OffsiteKeyRecoverer, error) { return rec, nil })
// R-216: the capability gate FAILS CLOSED, so the default fixture states the supported case
// explicitly. Without this every unlock test would exercise the refusal instead — which is exactly
// the protection working, and exactly not what those tests are about. The refusal has its own
// tests in recovery_gate_test.go, each overriding this.
s.SetRecoverySupport(func(context.Context) agentapi.SupportState { return agentapi.SupportYes })
return &recoveryFixture{s: s, mgr: mgr, sett: sett, rec: rec, runner: rr, dataDir: cfg.Paths.DataDir}
}
// placeRepoPassword makes the box look HEALTHY (it holds its own repository password).
func (f *recoveryFixture) placeRepoPassword(t *testing.T) {
t.Helper()
if err := f.mgr.InjectOffboxPassword(testRepoPW, true); err != nil {
t.Fatal(err)
}
}
func getRecoveryPage(t *testing.T, s *Server) *httptest.ResponseRecorder {
t.Helper()
rr := httptest.NewRecorder()
s.recoveryPageHandler(rr, httptest.NewRequest(http.MethodGet, "/recovery", nil))
return rr
}
// SCENARIO A — the page appears for the fresh + package box, and interrupts the landing pages.
func TestRecovery_A_PageAppearsForARebuiltBox(t *testing.T) {
f := newRecoveryFixture(t)
if !f.s.recoveryOffer() {
t.Fatal("a rebuilt box (fresh data area + a hub-held package) is not offered the recovery screen")
}
if !f.s.recoveryInterrupts() {
t.Fatal("the full page must interrupt the landing pages before any postpone")
}
rr := getRecoveryPage(t, f.s)
if rr.Code != http.StatusOK {
t.Fatalf("GET /recovery = %d", rr.Code)
}
body := rr.Body.String()
// The two MANDATORY sentences of §8.2 (ASCII-safe fragments — accented patterns get mangled
// through the ssh→pct chain and a false 0 reads exactly like the sentence being gone).
if !strings.Contains(body, "helyre") || !strings.Contains(body, "llít") {
t.Error("the page does not mention the recovery code at all")
}
if !strings.Contains(body, "senki nem tudja p") {
t.Error("MANDATORY: the page must say that nobody can replace a lost recovery code")
}
if !strings.Contains(body, "semmi nem v") {
t.Error("MANDATORY: the page must say that nothing is restored or changed in this step")
}
// It takes the code in a POST body, and the field does not autofill.
if !strings.Contains(body, `action="/recovery/unlock"`) || !strings.Contains(body, `method="POST"`) {
t.Error("the code form must POST to /recovery/unlock")
}
if !strings.Contains(body, `autocomplete="off"`) {
t.Error("the recovery-code field must not autofill")
}
// And it is not cached.
if cc := rr.Header().Get("Cache-Control"); !strings.Contains(cc, "no-store") {
t.Errorf("the recovery page must be no-store, got %q", cc)
}
}
// SCENARIO B — it does NOT appear for anyone else. THE GUARD ON THE CONJUNCTION.
//
// RED-PROOF: drop the hub-package condition from backup.OffsiteRecoveryOffer → the
// "never had off-site backups" case below FAILS, i.e. a brand-new customer is greeted on day one by
// a recovery screen for data they never had. That is the plausible wrong fix.
func TestRecovery_B_DoesNotAppearForAnyoneElse(t *testing.T) {
t.Run("healthy box (holds its own repository password)", func(t *testing.T) {
f := newRecoveryFixture(t)
f.placeRepoPassword(t) // healthy, and not orphaned
if f.s.recoveryOffer() {
t.Fatal("a HEALTHY box was offered the recovery screen")
}
})
t.Run("never had off-site backups (no hub package)", func(t *testing.T) {
f := newRecoveryFixture(t)
if err := f.sett.SetHubEscrowIdentityPresent(false); err != nil {
t.Fatal(err)
}
if f.s.recoveryOffer() {
t.Fatal("a box that never had off-site backups was offered a recovery screen for data it never had")
}
})
t.Run("not claimed — the page is behind the household password", func(t *testing.T) {
f := newRecoveryFixture(t)
f.s.cfg.Web.PasswordHash = "" // unclaimed: no password anywhere
if f.s.authEnabled() {
t.Fatal("fixture error: the box still reads as claimed")
}
// The interception is inside the authenticated surface: RequireAuth gates /launcher and
// /dashboard before ServeHTTP ever runs. Assert that through the REAL middleware chain.
mux := http.NewServeMux()
mux.Handle("/", f.s.RequireAuth(f.s.CsrfProtect(http.HandlerFunc(f.s.ServeHTTP))))
rr := httptest.NewRecorder()
mux.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/launcher", nil))
if rr.Code == http.StatusFound && rr.Header().Get("Location") == "/recovery" {
t.Fatal("an UNCLAIMED box redirected to the recovery screen — it shows metadata that belongs behind the household password")
}
})
}
// SCENARIO C — the correct code unlocks, places the key, and the page then shows what is in there.
func TestRecovery_C_UnlockOpensAndLists(t *testing.T) {
f := newRecoveryFixture(t)
now := time.Now().UTC()
f.runner.snapshots = []map[string]any{
{"short_id": "aaa1111", "time": now.Add(-24 * time.Hour).Format(time.RFC3339), "tags": []string{"immich"}},
{"short_id": "bbb2222", "time": now.Format(time.RFC3339), "tags": []string{"immich"}},
{"short_id": "ccc3333", "time": now.Add(-48 * time.Hour).Format(time.RFC3339), "tags": []string{"calibre-web"}},
}
rr := postUnlock(t, f.s, testRecoveryCode)
if rr.Code != http.StatusOK {
t.Fatalf("unlock = %d", rr.Code)
}
// EFFECT 1: the recovered key is on disk.
got, present := f.mgr.OffboxRepoPasswordHash()
if !present || got != backup.HashResticPassword(testRepoPW) {
t.Fatalf("the recovered repository password was not placed (present=%v)", present)
}
// EFFECT 2: the handler passed the code it was given, unmodified.
if len(f.rec.codes) != 1 || f.rec.codes[0] != testRecoveryCode {
t.Fatalf("the handler did not hand the agent the typed code: %+v", f.rec.codes)
}
// EFFECT 3: the page lists what is in there — apps and dates. A success message with nothing
// shown is indistinguishable from having unlocked an EMPTY store.
body := rr.Body.String()
for _, want := range []string{"immich", "calibre-web"} {
if !strings.Contains(body, want) {
t.Errorf("the listing does not name %q: the customer cannot tell whether this is their data", want)
}
}
if !strings.Contains(body, "4.0 MB") && !strings.Contains(body, "MB") {
t.Errorf("the listing shows no size")
}
// EFFECT 4: it did NOT restore anything — the page points at the restore page rather than doing it.
if !strings.Contains(body, "/backups/restore") {
t.Error("the page must point at the per-app restore rather than restoring")
}
if strings.Contains(body, "/backup/offbox/reconstitute") || strings.Contains(body, "/backup/offbox/place") {
t.Fatal("the recovery page offers a RESTORE action — unlocking and restoring are separate")
}
}
// The EMPTY store is stated plainly rather than shown as a bare list.
func TestRecovery_C_EmptyStoreSaysSo(t *testing.T) {
f := newRecoveryFixture(t)
f.runner.snapshots = nil // opened cleanly, holds nothing
body := postUnlock(t, f.s, testRecoveryCode).Body.String()
if !strings.Contains(body, "nincs benne egyetlen ment") {
t.Fatalf("an empty store must say so plainly — silence there reads as a broken page. body=%.400q", body)
}
}
// SCENARIO D — a wrong code fails closed, writes nothing, says what to check, and does NOT lock out.
func TestRecovery_D_WrongCodeFailsClosedAndIsKind(t *testing.T) {
f := newRecoveryFixture(t)
f.rec.fail = true
for i := 0; i < 6; i++ { // well past any plausible lockout threshold
rr := postUnlock(t, f.s, "rossz kod")
if rr.Code != http.StatusOK {
t.Fatalf("attempt %d: got %d, want a re-rendered page", i, rr.Code)
}
body := rr.Body.String()
if !strings.Contains(body, "nem fogadtuk el") {
t.Fatalf("attempt %d: the page does not say the code was not accepted: %.300q", i, body)
}
if !strings.Contains(body, "z szót") && !strings.Contains(body, "t sz") {
t.Errorf("attempt %d: the message does not say what to check", i)
}
// The raw agent error must NOT be shown to the customer.
if strings.Contains(body, "age:") || strings.Contains(body, "incorrect passphrase") {
t.Errorf("attempt %d: the raw technical error was rendered to the customer", i)
}
// NOTHING was written.
if _, present := f.mgr.OffboxRepoPasswordHash(); present {
t.Fatalf("attempt %d: a repository password was written on a FAILED unlock", i)
}
// And the form is still there — no lockout.
if !strings.Contains(body, `name="recovery_code"`) {
t.Fatalf("attempt %d: the customer was locked out of their own data after a mistyped code", i)
}
}
}
// SCENARIO E — "most nem" stops the interruption and NOTHING else. The entry point survives.
//
// RED-PROOF: bind the backups-page entry point to recoveryInterrupts instead of recoveryOffer → the
// route to the data disappears after one click.
func TestRecovery_E_PostponeKeepsTheEntryPoint(t *testing.T) {
f := newRecoveryFixture(t)
rr := httptest.NewRecorder()
f.s.recoveryPostponeHandler(rr, httptest.NewRequest(http.MethodPost, "/recovery/postpone", nil))
if rr.Code != http.StatusFound {
t.Fatalf("postpone = %d, want a redirect", rr.Code)
}
if !f.sett.GetRecoveryNoticePostponed() {
t.Fatal("the postpone was not recorded")
}
// The full page no longer interrupts…
if f.s.recoveryInterrupts() {
t.Fatal("the full page still interrupts after 'most nem'")
}
// …and it must NOT interrupt through the real mux either.
mux := http.NewServeMux()
mux.Handle("/", http.HandlerFunc(f.s.ServeHTTP))
rr2 := httptest.NewRecorder()
mux.ServeHTTP(rr2, httptest.NewRequest(http.MethodGet, "/launcher", nil))
if rr2.Code == http.StatusFound && rr2.Header().Get("Location") == "/recovery" {
t.Fatal("the landing page still redirects to the recovery screen after 'most nem'")
}
// …but the ENTRY POINT is untouched: the offer stands, so the backups page still renders it.
if !f.s.recoveryOffer() {
t.Fatal("'most nem' removed the OFFER — the customer has lost the route to their own data")
}
// And the page itself is still reachable directly.
if getRecoveryPage(t, f.s).Code != http.StatusOK {
t.Fatal("the recovery page is unreachable after 'most nem'")
}
// THE ENTRY POINT ITSELF, rendered: the backups page must still carry the route. Asserted on the
// rendered flag rather than on recoveryOffer alone, because the defect this guards against is the
// TEMPLATE being fed the wrong predicate.
rr3 := httptest.NewRecorder()
f.s.backupsRemoteHandler(rr3, httptest.NewRequest(http.MethodGet, "/backups/remote", nil))
if !strings.Contains(rr3.Body.String(), `href="/recovery"`) {
t.Fatal("the backups page no longer offers the route to the recovery screen after 'most nem' — a customer who clicked past it once has lost the way to their own data")
}
}
// SCENARIO F — "I do not want the old data" is confirmed TWICE and reaches the SHIPPED move-aside.
//
// RED-PROOF: render the final button on the first view (drop the ConfirmSetAside gate) → one click
// suffices, and this test fails on the first assertion.
func TestRecovery_F_SetAsideNeedsTwoConfirmations(t *testing.T) {
f := newRecoveryFixture(t)
// The move-aside only exists once the tier is orphaned — that is the shipped handler's own
// precondition, and the page only offers the choice when it can actually run.
if err := f.sett.SetOffboxTarget(&settings.OffboxTarget{
Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo",
Schedule: "daily", EscrowState: "escrowed", RepoState: "orphaned",
}); err != nil {
t.Fatal(err)
}
if !f.mgr.OffboxOrphaned() {
t.Fatal("fixture: the tier is not orphaned, so the set-aside cannot be offered")
}
// FIRST VIEW — the destructive-looking button must NOT be present yet.
first := getRecoveryPage(t, f.s).Body.String()
if strings.Contains(first, `action="/backup/offbox/reset"`) {
t.Fatal("the set-aside form is on the FIRST view — one click would set the customer's history aside")
}
if !strings.Contains(first, "setaside=1") {
t.Fatal("the first view offers no route to the set-aside choice at all")
}
// And it must read as the exceptional path, not an equal third button.
if strings.Count(first, "btn-primary") > 1 {
t.Error("the set-aside is styled as an equal primary action")
}
// SECOND VIEW — now the confirmation, naming exactly what happens.
rr := httptest.NewRecorder()
f.s.recoveryPageHandler(rr, httptest.NewRequest(http.MethodGet, "/recovery?setaside=1", nil))
second := rr.Body.String()
if !strings.Contains(second, `action="/backup/offbox/reset"`) {
t.Fatal("the second view does not reach the SHIPPED move-aside handler")
}
if !strings.Contains(second, `name="confirm" value="1"`) {
t.Fatal("the second view does not carry the shipped handler's confirm gate")
}
// The copy must say SET ASIDE, not delete — the whole point of the ruling.
if !strings.Contains(second, "lretessz") {
t.Error("the confirmation does not say the backups are SET ASIDE")
}
if !strings.Contains(second, "nem t") {
t.Error("the confirmation does not say they are NOT deleted")
}
}
// SCENARIO G — the command line and the page drive ONE core.
//
// RED-PROOF: give the handler its own copy of fetch→compare→install instead of calling
// RecoverInstallCore → this test still passes on the happy path, so it asserts the SHARED SYMBOL
// from source (below) as well as the behaviour here.
func TestRecovery_G_PageAndCLIShareOneCore(t *testing.T) {
// Behavioural half: the same fake, the same outcome, through both callers.
f := newRecoveryFixture(t)
res, err := backup.RecoverInstallCore(context.Background(), f.mgr, f.rec, testRecoveryCode, true)
if err != nil || res.Outcome != backup.RecoverInstalled {
t.Fatalf("core install: outcome=%q err=%v", res.Outcome, err)
}
// Re-running is UNCHANGED, not a second install — the same three outcomes the CLI documents.
res2, err2 := backup.RecoverInstallCore(context.Background(), f.mgr, f.rec, testRecoveryCode, true)
if err2 != nil || res2.Outcome != backup.RecoverUnchanged {
t.Fatalf("core re-run: outcome=%q err=%v", res2.Outcome, err2)
}
}
// SCENARIO H (§8.3) — the recovery code persists NOWHERE, with the planted-copy positive control.
func TestRecovery_H_CodeLeavesNoTrace(t *testing.T) {
f := newRecoveryFixture(t)
var logBuf strings.Builder
f.s.logger = log.New(&logBuf, "", 0)
rr := postUnlock(t, f.s, testRecoveryCode)
// 1) not echoed in the response
if strings.Contains(rr.Body.String(), testRecoveryCode) {
t.Fatal("the recovery code was echoed back in the rendered page")
}
// 2) not in any log line
if strings.Contains(logBuf.String(), testRecoveryCode) {
t.Fatal("the recovery code reached the log")
}
// 3) not in any file under the data dir
found := grepTree(t, f.dataDir, testRecoveryCode)
if found != "" {
t.Fatalf("the recovery code was persisted to %s", found)
}
// THE POSITIVE CONTROL — a sweep whose sensitivity was never shown is not evidence. Plant a copy
// where the sweep looks and require it to be found; then remove it.
planted := filepath.Join(f.dataDir, "planted-control.txt")
if err := os.WriteFile(planted, []byte("x "+testRecoveryCode+" x"), 0o600); err != nil {
t.Fatal(err)
}
if got := grepTree(t, f.dataDir, testRecoveryCode); got == "" {
t.Fatal("POSITIVE CONTROL FAILED: the sweep could not find a planted copy, so its earlier silence proves nothing")
}
if err := os.Remove(planted); err != nil {
t.Fatal(err)
}
if got := grepTree(t, f.dataDir, testRecoveryCode); got != "" {
t.Fatalf("the control was not cleaned up: %s", got)
}
}
// grepTree returns the first file under root whose contents contain needle ("" when none).
func grepTree(t *testing.T, root, needle string) string {
t.Helper()
var hit string
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil || info == nil || info.IsDir() || hit != "" {
return nil
}
b, rerr := os.ReadFile(p)
if rerr == nil && strings.Contains(string(b), needle) {
hit = p
}
return nil
})
return hit
}
func postUnlock(t *testing.T, s *Server, code string) *httptest.ResponseRecorder {
t.Helper()
form := url.Values{"recovery_code": {code}}
req := httptest.NewRequest(http.MethodPost, "/recovery/unlock", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
rr := httptest.NewRecorder()
s.recoveryUnlockHandler(rr, req)
return rr
}