From e03b18ea211d052106dc60fe19df23564d04a47c Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 7 Oct 2026 10:07:50 +0200 Subject: [PATCH] R-32 (decision 167): RESET purges the off-site folder through the sub-account's own login before deleting it Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../architecture/07-backup-architecture.md | 10 +++ hub/CHANGELOG.md | 1 + hub/cmd/hub/main.go | 9 ++- hub/internal/offsite/offsite.go | 21 ++++- hub/internal/offsite/offsite_test.go | 1 + hub/internal/offsite/r32_purge_test.go | 70 ++++++++++++++++ hub/internal/offsitekeys/offsitekeys.go | 54 +++++++++++++ hub/internal/offsitekeys/r32_purge_test.go | 80 +++++++++++++++++++ hub/internal/offsitekeys/service.go | 16 ++++ hub/internal/web/customer_edit_tab_test.go | 2 + 10 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 hub/internal/offsite/r32_purge_test.go create mode 100644 hub/internal/offsitekeys/r32_purge_test.go diff --git a/documentation/architecture/07-backup-architecture.md b/documentation/architecture/07-backup-architecture.md index 9269b27d..1ff8fda7 100644 --- a/documentation/architecture/07-backup-architecture.md +++ b/documentation/architecture/07-backup-architecture.md @@ -245,6 +245,16 @@ the identity bundle's shape is `{tunnel_token, pbs_token, wg_private_key, restic **[DESIGN] Off-site deletion custody (decisions 68–69, 2026-10-03) — BUILT hub v0.127.0 / controller v0.289.1, live on both demo boxes.** The restic repository password stays on the box only. The box's off-site key is append-only (pinned in `authorized_keys`, written by the hub — the key registrar); the box never receives the sub-account password, which the hub stores encrypted at rest. Old snapshots are pruned by the box itself, only in a weekly window the hub opens, behind a fake-snapshot guard (R-822); the window has run live on both demo boxes since 2026-10-05 (R-95, closed). The guard's residual — past-dated fakes steering the keeps, and a window check that trusts the box's own counts — is R-822 and R-895. A Felhom-side pruner holding repository passwords is rejected. +**[DESIGN, built hub 2026-10-07 — R-32 option A, `09` §3 decision 167] What RESET does to the household's off-site +copy.** On the shared pool box a sub-account is a LOGIN, not the data: deleting it leaves its folder (measured +2026-07-21: 1.4 GB in two `.orphaned-*` folders after a RESET), and a re-enabled customer lands on the same folder +(`felhom-`). So RESET's off-site leg now **purges first**: through the sub-account's own password login +(the route `DeleteSetAside` already uses — no main-account credential), it deletes the repository and every +`.orphaned-*` copy and nothing else, lists again, and only then deletes the sub-account. A purge that fails (or +no purge route) keeps the sub-account and fails the leg (`hetzner: failed`, a re-run resumes). The dedicated tier is +unchanged (the box is deleted with its data). The **move-aside for a reinstall WITHOUT RESET stays** — there custody +survives and the set-aside copy is still openable. Tests `TestDeprovision_R32_*`, `TestPurgeRepos_R32_*`. + **[FACT] Three parts of the Recipe are empty or wrong on the live fleet**, and they are exactly the parts a host-loss recovery would read (INV Part D2.3): `hosts.dr_record_json` is `{}` on all three hosts; `host_escrow.directive_json` is `{}` on both escrowed hosts; `dr_recipe.host_half.drives` is diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 22355c03..a1768d8c 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -11,6 +11,7 @@ `documentation/audits/day-2026-10-07/B/red-hub-pve-mutations.txt`. ## Unreleased (2026-10-07) +- hub (R-32 option A, `09` §3 decision 167): RESET's off-site leg on the shared pool box purges the household's folder — the repository and every `.orphaned-*` copy, nothing else — through the sub-account's OWN login (`offsitekeys.Registrar.PurgeRepos`, the `DeleteSetAside` route) BEFORE it deletes the sub-account; a failed purge or no purge route keeps the sub-account and fails the leg (re-run resumes). It used to delete only the sub-account (a login), leaving the folder for the next lifecycle. The false "repo DATA dies with the sub-account" comment is corrected. Tests `TestDeprovision_R32_*`, `TestPurgeRepos_R32_*` (red-proved, `documentation/audits/day-2026-10-07/D/`). `07` §6 (RESET) updated. - hub (R-105 option A, `09` §3 decision 169): the two never-built "slim DR record" fields are retired. `hosts.dr_record_json` is no longer scanned (it had no writer and no reader); the escrow PUT no longer stores a `directive` (only a by-hand selftest flag ever sent one, and every wizard escrow overwrote it with `{}`); the re-enroll and restore-directive routes still serve the two opaque blobs and a `directive` of `{}`. The columns stay in the schema, unread. `TestR105_IdentitySaveLeavesDirectiveColumnAlone`, `TestR105_RetiredColumnsHaveNoReader` (red-proved, `documentation/audits/day-2026-10-07/F/`). `05` §9/§11 and `06` §3.5 corrected. ## v0.141.0 — a reinstall's new backup key no longer overwrites the old one in the escrow; a second off-site Save is refused, not raced; the SMART counters are modelled (R-366, R-31, R-330) (2026-10-07) diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 2eae95ec..da43229e 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -378,6 +378,7 @@ func main() { // a heal that cannot actually restage must never run (it would emit a restaged event for a // silent no-op: ReissueOffsiteForCustomer returns nil when offsite is unconfigured). var offsiteHealReissuer monitor.OffsiteReissuer + var offsiteProv *offsite.Provisioner // R-32: the key service wires its purge into RESET's Deprovision below if tok := os.Getenv("HETZNER_TOKEN"); tok != "" { poolBoxID, _ := strconv.ParseInt(os.Getenv("HETZNER_POOL_BOX_ID"), 10, 64) location := os.Getenv("HETZNER_LOCATION") @@ -385,9 +386,10 @@ func main() { location = "fsn1" } client := hetznerapi.NewClient(func() string { return os.Getenv("HETZNER_TOKEN") }) - webServer.SetOffsiteProvisioner(&offsite.Provisioner{ + offsiteProv = &offsite.Provisioner{ API: client, Store: dataStore, Scanner: offsite.SSHHostKeyScanner{}, PoolBoxID: poolBoxID, Location: location, Logger: logger, - }) + } + webServer.SetOffsiteProvisioner(offsiteProv) logger.Printf("[INFO] Offsite provisioning enabled (pool_box=%d, location=%s)", poolBoxID, location) offsiteHealReissuer = webServer // R-71c heal armed (provisioner present) // R-5 (v0.64.0): the pool-box aggregate checker shares the SAME client + pool box id (GET-only). @@ -542,6 +544,9 @@ func main() { } } apiHandler.SetOffsiteKeyService(keySvc) + if offsiteProv != nil { + offsiteProv.PurgeShared = keySvc.PurgeRepos // R-32: RESET purges the folder through the sub-account's own login + } runKeyAudit := func(ctx context.Context) any { start := time.Now() out := keySvc.AuditAll(ctx, dataStore.OffsiteWindowOpen) diff --git a/hub/internal/offsite/offsite.go b/hub/internal/offsite/offsite.go index da010f32..39d23dee 100644 --- a/hub/internal/offsite/offsite.go +++ b/hub/internal/offsite/offsite.go @@ -69,6 +69,12 @@ type Provisioner struct { // default ~60s ladder. Tests inject zeros. The total must fit inside applyOffsite's 3-min detached ctx. ScanBackoff []time.Duration + // PurgeShared (R-32 option A, `09` §3 decision 167) deletes the customer's off-site repository and every + // set-aside copy through the sub-account's OWN login (offsitekeys.Service.PurgeRepos — wired in main, a seam + // because offsitekeys imports this package). Deprovision calls it BEFORE deleting a shared sub-account; nil or a + // failure refuses the delete, because a deleted sub-account leaves its folder reachable only by the main account. + PurgeShared func(ctx context.Context, customerID string) error + // inflight (R-31) holds the customers whose provisioning is running RIGHT NOW. // A second Save while the first still runs used to race it: both listed no sub-account, both created // one (a second dedicated box is a second bill). The second call now fails at once with @@ -311,8 +317,11 @@ func (p *Provisioner) OffsiteIdentifier(ctx context.Context, customerID, typ str } } -// Deprovision DELETES the customer's offsite storage — the RESET teardown (v0.61.0). The offsite repo -// DATA dies with the sub-account/box; irreversible, gated by the operator RESET confirm. IDEMPOTENT: +// Deprovision DELETES the customer's offsite storage — the RESET teardown (v0.61.0); irreversible, gated by the +// operator RESET confirm. Dedicated tier: the box is deleted, and its data with it. Shared tier: a sub-account is a +// LOGIN, not the data — its folder survives the sub-account's deletion (R-32, measured 2026-07-21: 1.4 GB left in two +// `.orphaned-*` folders), so the folder is purged through PurgeShared FIRST, and the sub-account is deleted only after +// that succeeds (TestDeprovision_R32_*). IDEMPOTENT: // zero labelled sub-accounts/boxes = already gone = success (a re-run after a partial reset does not // error). The id is re-derived by the customer label each time — nothing stored to go stale. func (p *Provisioner) Deprovision(ctx context.Context, customerID, typ string) error { @@ -349,6 +358,12 @@ func (p *Provisioner) Deprovision(ctx context.Context, customerID, typ string) e p.logf("[offsite] deprovision: no sub-account labelled for %s — already gone", customerID) return nil } + if p.PurgeShared == nil { + return fmt.Errorf("offsite: no purge route for %s's shared folder — refusing to delete the sub-account (its folder would be left behind, reachable only by the main account)", customerID) + } + if err := p.PurgeShared(ctx, customerID); err != nil { + return fmt.Errorf("offsite: purge %s's off-site folder before deleting the sub-account: %w (the sub-account is kept; re-run RESET to resume)", customerID, err) + } for _, sub := range subs { act, err := p.API.DeleteSubaccount(ctx, p.PoolBoxID, sub.ID) if err != nil { @@ -357,7 +372,7 @@ func (p *Provisioner) Deprovision(ctx context.Context, customerID, typ string) e if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: delete sub-account action: %w", err) } - p.logf("[offsite] deprovisioned shared sub-account %d for %s (repo data destroyed)", sub.ID, customerID) + p.logf("[offsite] deprovisioned shared sub-account %d for %s (its folder was purged first)", sub.ID, customerID) } return nil } diff --git a/hub/internal/offsite/offsite_test.go b/hub/internal/offsite/offsite_test.go index 456222d8..7cadf5bb 100644 --- a/hub/internal/offsite/offsite_test.go +++ b/hub/internal/offsite/offsite_test.go @@ -446,6 +446,7 @@ func TestDeprovision_SharedIdempotent(t *testing.T) { if fake.CreatedSubaccounts != 1 { t.Fatalf("precondition: want 1 subaccount, got %d", fake.CreatedSubaccounts) } + p.PurgeShared = func(context.Context, string) error { return nil } // R-32: the folder purge is pinned in r32_purge_test.go if err := p.Deprovision(context.Background(), "cust-d", "shared"); err != nil { t.Fatalf("deprovision: %v", err) } diff --git a/hub/internal/offsite/r32_purge_test.go b/hub/internal/offsite/r32_purge_test.go new file mode 100644 index 00000000..45cdeea5 --- /dev/null +++ b/hub/internal/offsite/r32_purge_test.go @@ -0,0 +1,70 @@ +package offsite + +import ( + "context" + "errors" + "testing" + + "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" +) + +// orderAPI records when the sub-account is deleted relative to the purge. +type orderAPI struct { + *hetznerapi.Fake + log *[]string +} + +func (o *orderAPI) DeleteSubaccount(ctx context.Context, boxID, subID int64) (hetznerapi.Action, error) { + *o.log = append(*o.log, "delete-subaccount") + return o.Fake.DeleteSubaccount(ctx, boxID, subID) +} + +// R-32 option A (`09` §3 decision 167) — RESET's shared-tier teardown purges the folder through the sub-account's +// own login BEFORE the sub-account is deleted; a sub-account is a login, its folder survives its deletion. +// +// COMPANION RED-PROOF (observed): remove the PurgeShared call from Deprovision → this fails with +// "the folder must be purged before the sub-account is deleted; order [delete-subaccount]". Restored. +func TestDeprovision_R32_PurgesFolderBeforeDeletingSubaccount(t *testing.T) { + p, fake, _ := newTestProvisioner(t) + if _, err := p.ProvisionOffsite(context.Background(), "cust-r32", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { + t.Fatal(err) + } + var order []string + p.API = &orderAPI{Fake: fake, log: &order} + p.PurgeShared = func(_ context.Context, id string) error { order = append(order, "purge:"+id); return nil } + if err := p.Deprovision(context.Background(), "cust-r32", "shared"); err != nil { + t.Fatal(err) + } + if len(order) != 2 || order[0] != "purge:cust-r32" || order[1] != "delete-subaccount" { + t.Fatalf("the folder must be purged before the sub-account is deleted; order %v", order) + } +} + +// A failed purge keeps the sub-account (after its deletion only the main account reaches the folder), and the +// error reaches RESET (its hetzner leg reads "failed" and a re-run resumes). +// COMPANION RED-PROOF (observed): ignore PurgeShared's error → this fails with "a failed purge must keep the +// sub-account; deleted=1". Restored. +func TestDeprovision_R32_FailedPurgeKeepsSubaccount(t *testing.T) { + p, fake, _ := newTestProvisioner(t) + if _, err := p.ProvisionOffsite(context.Background(), "cust-r32b", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { + t.Fatal(err) + } + p.PurgeShared = func(context.Context, string) error { return errors.New("permission denied") } + if err := p.Deprovision(context.Background(), "cust-r32b", "shared"); err == nil { + t.Fatal("a failed purge must fail Deprovision") + } + if fake.DeletedSubaccounts != 0 { + t.Fatalf("a failed purge must keep the sub-account; deleted=%d", fake.DeletedSubaccounts) + } +} + +// No purge route wired → refuse; never a silent delete that strands the folder. +func TestDeprovision_R32_NoPurgeRouteRefuses(t *testing.T) { + p, fake, _ := newTestProvisioner(t) + if _, err := p.ProvisionOffsite(context.Background(), "cust-r32c", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { + t.Fatal(err) + } + if err := p.Deprovision(context.Background(), "cust-r32c", "shared"); err == nil || fake.DeletedSubaccounts != 0 { + t.Fatalf("no purge route must refuse and keep the sub-account; err=%v deleted=%d", err, fake.DeletedSubaccounts) + } +} diff --git a/hub/internal/offsitekeys/offsitekeys.go b/hub/internal/offsitekeys/offsitekeys.go index 25b72245..cc84265f 100644 --- a/hub/internal/offsitekeys/offsitekeys.go +++ b/hub/internal/offsitekeys/offsitekeys.go @@ -442,6 +442,60 @@ func (r *Registrar) DeleteSetAside(ctx context.Context, t Target, password, path return nil } +// PurgeRepos (R-32 option A, `09` §3 decision 167) deletes the household's off-site repository AND every set-aside +// copy (`.orphaned-*`) through the sub-account's OWN login — RESET's off-site leg, run BEFORE the sub-account is +// removed (after that only the pool box's main account could reach the folder). It deletes nothing else: it lists the +// repository's parent directory and removes only the repository itself and IsSetAsidePath matches. It then lists +// again and fails if any of them is still there. Returns the paths it removed (none = nothing was there). +func (r *Registrar) PurgeRepos(ctx context.Context, t Target, password string) ([]string, error) { + sh, err := r.open(ctx, t, password) + if err != nil { + return nil, err + } + defer sh.Close() + targets, err := repoAndSetAsides(ctx, sh, t.RepoPath) + if err != nil { + return nil, err + } + for _, p := range targets { + if p != t.RepoPath && !IsSetAsidePath(t.RepoPath, p) { + return nil, fmt.Errorf("offsitekeys: %q is neither the repository nor a set-aside copy — refusing", p) + } + if _, err := sh.Run(ctx, "rm -rf "+p, nil); err != nil { + return nil, fmt.Errorf("offsitekeys: purge %s: %w", p, err) + } + } + left, err := repoAndSetAsides(ctx, sh, t.RepoPath) + if err != nil { + return nil, fmt.Errorf("offsitekeys: re-list after the purge: %w", err) + } + if len(left) > 0 { + return nil, fmt.Errorf("offsitekeys: still present after the purge: %s", strings.Join(left, " ")) + } + return targets, nil +} + +// repoAndSetAsides lists the repository's parent directory and returns the repository and its set-aside copies. +func repoAndSetAsides(ctx context.Context, sh Shell, repo string) ([]string, error) { + i := strings.LastIndex(repo, "/") + if repo == "" || i <= 0 { + return nil, fmt.Errorf("offsitekeys: repository path %q has no parent directory", repo) + } + dir := repo[:i] + out, err := sh.Run(ctx, "ls "+dir, nil) + if err != nil { + return nil, fmt.Errorf("offsitekeys: list %s: %w", dir, err) + } + var found []string + for _, name := range strings.Fields(string(out)) { + p := dir + "/" + strings.TrimSuffix(name, "/") + if p == repo || IsSetAsidePath(repo, p) { + found = append(found, p) + } + } + return found, nil +} + // IsSetAsidePath: `.orphaned-` with a suffix of [A-Za-z0-9-] only. func IsSetAsidePath(repo, path string) bool { pre := repo + ".orphaned-" diff --git a/hub/internal/offsitekeys/r32_purge_test.go b/hub/internal/offsitekeys/r32_purge_test.go new file mode 100644 index 00000000..45ca5f3d --- /dev/null +++ b/hub/internal/offsitekeys/r32_purge_test.go @@ -0,0 +1,80 @@ +package offsitekeys + +import ( + "context" + "errors" + "sort" + "strings" + "testing" +) + +// dirFS: a folder listing for `ls ` and `rm -rf `, as the provider's restricted shell answers them. +type dirFS struct { + entries map[string]bool // full paths under /home + cmds []string + stick string // a path whose rm "succeeds" but stays (the re-list must catch it) +} + +func (f *dirFS) Run(_ context.Context, cmd string, _ []byte) ([]byte, error) { + f.cmds = append(f.cmds, cmd) + a := strings.Fields(cmd) + switch { + case a[0] == "ls" && len(a) == 2: + var names []string + for p := range f.entries { + if strings.HasPrefix(p, a[1]+"/") { + names = append(names, strings.TrimPrefix(p, a[1]+"/")) + } + } + sort.Strings(names) + return []byte(strings.Join(names, "\n") + "\n"), nil + case a[0] == "rm" && a[1] == "-rf": + if a[2] != f.stick { + delete(f.entries, a[2]) + } + return nil, nil + } + return nil, errors.New("Command not found") +} +func (f *dirFS) Close() error { return nil } + +type dirDialer struct{ fs *dirFS } + +func (d dirDialer) Dial(context.Context, Target, string) (Shell, error) { return d.fs, nil } + +// R-32 — the purge removes the repository and every set-aside copy, and NOTHING else in the folder. +// COMPANION RED-PROOF (observed): drop the IsSetAsidePath filter (remove every listed entry) → this fails with +// "want the repo + 2 set-asides removed, got [… /home/felhom-repo-notes … /home/other]". Restored. +func TestPurgeRepos_R32_RemovesRepoAndSetAsidesOnly(t *testing.T) { + fs := &dirFS{entries: map[string]bool{ + "/home/felhom-repo": true, "/home/felhom-repo.orphaned-20260721": true, "/home/felhom-repo.orphaned-20260721-2": true, + "/home/felhom-repo-notes": true, "/home/other": true, + }} + removed, err := (&Registrar{Dialer: dirDialer{fs}}).PurgeRepos(context.Background(), tgt, "pw") + if err != nil { + t.Fatal(err) + } + if len(removed) != 3 { + t.Fatalf("want the repo + 2 set-asides removed, got %v", removed) + } + if !fs.entries["/home/felhom-repo-notes"] || !fs.entries["/home/other"] || len(fs.entries) != 2 { + t.Fatalf("other entries must stay: %v", fs.entries) + } +} + +// A copy that survives the delete fails the purge (so RESET keeps the sub-account). +func TestPurgeRepos_R32_SurvivorFails(t *testing.T) { + fs := &dirFS{entries: map[string]bool{"/home/felhom-repo": true}, stick: "/home/felhom-repo"} + if _, err := (&Registrar{Dialer: dirDialer{fs}}).PurgeRepos(context.Background(), tgt, "pw"); err == nil { + t.Fatal("a repository still present after the purge must fail it") + } +} + +// An empty folder (already purged, e.g. a re-run) is success with nothing removed. +func TestPurgeRepos_R32_EmptyIsSuccess(t *testing.T) { + fs := &dirFS{entries: map[string]bool{}} + removed, err := (&Registrar{Dialer: dirDialer{fs}}).PurgeRepos(context.Background(), tgt, "pw") + if err != nil || len(removed) != 0 { + t.Fatalf("an empty folder is success; got %v %v", removed, err) + } +} diff --git a/hub/internal/offsitekeys/service.go b/hub/internal/offsitekeys/service.go index eda79897..42256e8a 100644 --- a/hub/internal/offsitekeys/service.go +++ b/hub/internal/offsitekeys/service.go @@ -147,6 +147,22 @@ func (s *Service) MoveAside(ctx context.Context, customerID string) (string, err return name, nil } +// PurgeRepos (R-32 option A) is RESET's off-site purge: the customer's repository and every set-aside copy, deleted +// through the sub-account's own login, before the sub-account is removed. Fails closed: no stored credential, an +// unreadable folder or a copy that survives → an error, and the caller must NOT delete the sub-account. +func (s *Service) PurgeRepos(ctx context.Context, customerID string) error { + t, pw, err := s.TargetFor(customerID) + if err != nil { + return err + } + removed, err := s.Reg.PurgeRepos(ctx, t, pw) + if err != nil { + return err + } + s.logf("[WARN] offsitekeys: RESET purged %s's off-site folder through its own login: %d path(s) %v", customerID, len(removed), removed) + return nil +} + // AuditOutcome is one customer's daily-check result. type AuditOutcome struct { CustomerID string diff --git a/hub/internal/web/customer_edit_tab_test.go b/hub/internal/web/customer_edit_tab_test.go index 90a5ad79..95ffd14c 100644 --- a/hub/internal/web/customer_edit_tab_test.go +++ b/hub/internal/web/customer_edit_tab_test.go @@ -5,6 +5,7 @@ package web // contract of the surrounding POST actions. Groups A + B of the task's test plan. import ( + "context" "io" "log" "net/http" @@ -221,6 +222,7 @@ func TestCustomerActions_RedirectAnchors(t *testing.T) { s.SetOffsiteProvisioner(&offsite.Provisioner{ API: hetznerapi.NewFake(), Store: st, Scanner: webTestScanner{}, PoolBoxID: 611714, Location: "fsn1", Logger: log.New(io.Discard, "", 0), + PurgeShared: func(context.Context, string) error { return nil }, // R-32: pinned in internal/offsite }) cfg := &store.CustomerConfig{ CustomerID: "c1", CustomerName: "Acme", Domain: "acme.hu",