hub v0.39.0: offsite hardening — F4 credential re-issue, F2 scan retry, F5 save UX

F4: ReissueCredentials — explicit operator recovery for consumed-password dead-ends; resets the labelled resource's password (exactly-1 guard, red-proofed), stores a fresh one-time secret, bumps ConfigVersion. New hetznerapi.ResetBoxPassword for the dedicated path. F2: host-key scan retry-with-backoff (~60s ladder, red-proofed) — first save survives fresh-subaccount DNS lag. F5: config form disables submits + shows an in-flight notice (the re-click bait that caused live F1). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-09 22:39:08 +02:00
parent ecf9185605
commit 17cc67f7cd
9 changed files with 321 additions and 3 deletions
@@ -16,6 +16,7 @@ import (
 	"log"
 	"math/big"
 	"strings"
+	"time"

 	"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
@@ -61,8 +62,15 @@ type Provisioner struct {
 	PoolBoxID int64          // the shared-pool storage-box id (e.g. 611421)
 	Location  string         // dedicated-box location, e.g. "fsn1"
 	Logger    *log.Logger
+	// ScanBackoff is the retry schedule for the host-key scan (F2: a fresh sub-account's DNS name lags
+	// creation by seconds-to-a-minute, so the first scan typically fails with "no such host"). nil → the
+	// default ~60s ladder. Tests inject zeros. The total must fit inside applyOffsite's 3-min detached ctx.
+	ScanBackoff []time.Duration
 }

+// defaultScanBackoff: 5 retries, ~60s total — sized to the observed DNS propagation lag.
+var defaultScanBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second}
+
 func (p *Provisioner) logf(f string, a ...any) {
 	if p.Logger != nil {
 		p.Logger.Printf(f, a...)
@@ -106,7 +114,7 @@ func (p *Provisioner) ProvisionOffsite(ctx context.Context, customerID string, i
 	if port == 0 {
 		port = sftpPort
 	}
-	fp, err := p.Scanner.Fingerprint(ctx, d.Host, port)
+	fp, err := p.scanWithRetry(ctx, d.Host, port)
 	if err != nil {
 		return nil, fmt.Errorf("offsite: host-key scan %s: %w", d.Host, err)
 	}
@@ -114,6 +122,81 @@ func (p *Provisioner) ProvisionOffsite(ctx context.Context, customerID string, i
 	return d, nil
 }

+// scanWithRetry retries the host-key scan on failure (F2: fresh-resource DNS lag). Fail-closed past the
+// budget; ctx cancellation aborts between attempts.
+func (p *Provisioner) scanWithRetry(ctx context.Context, host string, port int) (string, error) {
+	backoff := p.ScanBackoff
+	if backoff == nil {
+		backoff = defaultScanBackoff
+	}
+	fp, err := p.Scanner.Fingerprint(ctx, host, port)
+	for i := 0; err != nil && i < len(backoff); i++ {
+		p.logf("[offsite] host-key scan %s failed (attempt %d/%d, retrying in %s): %v", host, i+1, len(backoff)+1, backoff[i], err)
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case <-time.After(backoff[i]):
+		}
+		fp, err = p.Scanner.Fingerprint(ctx, host, port)
+	}
+	return fp, err
+}
+
+// ReissueCredentials resets the customer's offsite credential and stores a FRESH one-time password — the
+// EXPLICIT operator recovery for a consumed-password dead-end (a fresh guest at DR, or a
+// consumed-but-failed install). It is NOT implicit rotation: ProvisionOffsite never calls this. Scoped
+// hard: the reset targets ONLY the resource labelled `felhom-customer=<id>`, and refuses unless the label
+// lookup finds exactly one. The password value is never logged (the action is).
+func (p *Provisioner) ReissueCredentials(ctx context.Context, customerID, typ string) error {
+	pw, err := genPassword()
+	if err != nil {
+		return err
+	}
+	switch typ {
+	case "shared":
+		if p.PoolBoxID == 0 {
+			return fmt.Errorf("offsite: no shared pool box configured")
+		}
+		subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID))
+		if err != nil {
+			return fmt.Errorf("offsite: reissue lookup: %w", err)
+		}
+		if len(subs) != 1 {
+			return fmt.Errorf("offsite: reissue needs exactly 1 sub-account labelled for %s, found %d — refusing", customerID, len(subs))
+		}
+		act, err := p.API.ResetSubaccountPassword(ctx, p.PoolBoxID, subs[0].ID, pw)
+		if err != nil {
+			return fmt.Errorf("offsite: reset sub-account password: %w", err)
+		}
+		if err := p.API.WaitAction(ctx, act); err != nil {
+			return fmt.Errorf("offsite: reset action: %w", err)
+		}
+		p.logf("[offsite] re-issued shared credentials for %s (subaccount %d)", customerID, subs[0].ID)
+	case "dedicated":
+		boxes, err := p.API.ListStorageBoxes(ctx, customerSelector(customerID))
+		if err != nil {
+			return fmt.Errorf("offsite: reissue lookup: %w", err)
+		}
+		if len(boxes) != 1 {
+			return fmt.Errorf("offsite: reissue needs exactly 1 box labelled for %s, found %d — refusing", customerID, len(boxes))
+		}
+		act, err := p.API.ResetBoxPassword(ctx, boxes[0].ID, pw)
+		if err != nil {
+			return fmt.Errorf("offsite: reset box password: %w", err)
+		}
+		if err := p.API.WaitAction(ctx, act); err != nil {
+			return fmt.Errorf("offsite: reset action: %w", err)
+		}
+		p.logf("[offsite] re-issued dedicated credentials for %s (box %d)", customerID, boxes[0].ID)
+	default:
+		return fmt.Errorf("offsite: reissue: unknown type %q (want shared|dedicated)", typ)
+	}
+	if err := p.Store.SaveOneTimeSecret(customerID, pw); err != nil {
+		return fmt.Errorf("offsite: store re-issued one-time password: %w", err)
+	}
+	return nil
+}
+
 func (p *Provisioner) provisionShared(ctx context.Context, customerID string, in Input) (*Descriptor, error) {
 	if p.PoolBoxID == 0 {
 		return nil, fmt.Errorf("offsite: no shared pool box configured")
@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"

 	"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
@@ -23,7 +24,24 @@ func newTestProvisioner(t *testing.T) (*Provisioner, *hetznerapi.Fake, *store.St
 	}
 	t.Cleanup(func() { st.Close() })
 	fake := hetznerapi.NewFake()
-	return &Provisioner{API: fake, Store: st, Scanner: &fakeScanner{fp: "SHA256:testfp"}, PoolBoxID: 611421, Location: "fsn1", Logger: log.New(io.Discard, "", 0)}, fake, st
+	// ScanBackoff: empty (non-nil) → single scan attempt, no retries — tests that want the F2 retry set
+	// their own schedule (nil would select the ~60s production default and stall the suite).
+	return &Provisioner{API: fake, Store: st, Scanner: &fakeScanner{fp: "SHA256:testfp"}, PoolBoxID: 611421, Location: "fsn1", Logger: log.New(io.Discard, "", 0), ScanBackoff: []time.Duration{}}, fake, st
+}
+
+// flakyScanner fails the first `failures` calls (fresh-resource DNS lag), then succeeds.
+type flakyScanner struct {
+	failures int
+	fp       string
+	calls    int
+}
+
+func (f *flakyScanner) Fingerprint(_ context.Context, _ string, _ int) (string, error) {
+	f.calls++
+	if f.calls <= f.failures {
+		return "", errors.New("dial tcp: lookup fresh.your-storagebox.de: no such host")
+	}
+	return f.fp, nil
 }

 // fakeScanner returns a fixed fingerprint (or an error) — no live SSH in tests.
@@ -96,6 +114,103 @@ func TestProvision_ScanFailClosed(t *testing.T) {
 	_ = st
 }

+// Scenario D (F2) — a fresh sub-account's DNS lags creation: the scan is retried and the FIRST save
+// serves the descriptor.
+func TestProvision_ScanRetriesThroughDNSLag(t *testing.T) {
+	p, _, _ := newTestProvisioner(t)
+	p.ScanBackoff = []time.Duration{0, 0, 0} // instant retries in tests
+	sc := &flakyScanner{failures: 2, fp: "SHA256:late"}
+	p.Scanner = sc
+	d, err := p.ProvisionOffsite(context.Background(), "cust-dns", Input{Enabled: true, Type: "shared", QuotaGB: 10})
+	if err != nil {
+		t.Fatalf("the first save must survive DNS lag via scan retries, got: %v", err)
+	}
+	if d.HostFingerprint != "SHA256:late" || sc.calls != 3 {
+		t.Fatalf("want fingerprint after 2 failed + 1 good scan, got fp=%q calls=%d", d.HostFingerprint, sc.calls)
+	}
+}
+
+// Scenario D (F2) — a scan that keeps failing past the budget still fails CLOSED (no descriptor).
+func TestProvision_ScanExhaustedFailsClosed(t *testing.T) {
+	p, _, _ := newTestProvisioner(t)
+	p.ScanBackoff = []time.Duration{0, 0}
+	sc := &flakyScanner{failures: 99, fp: "SHA256:never"}
+	p.Scanner = sc
+	d, err := p.ProvisionOffsite(context.Background(), "cust-dns2", Input{Enabled: true, Type: "shared", QuotaGB: 10})
+	if err == nil || d != nil {
+		t.Fatalf("an exhausted scan budget must fail closed, got d=%+v err=%v", d, err)
+	}
+	if sc.calls != 3 { // 1 initial + 2 retries
+		t.Fatalf("want 3 attempts (1 + 2 retries), got %d", sc.calls)
+	}
+}
+
+// Scenario A (F4) — re-issue resets the labelled sub-account's password and stores a FRESH one-time
+// secret (the consumed-password dead-end recovery).
+func TestReissue_SharedFreshSecret(t *testing.T) {
+	p, fake, st := newTestProvisioner(t)
+	if _, err := p.ProvisionOffsite(context.Background(), "cust-r", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil {
+		t.Fatal(err)
+	}
+	pw1, err := st.ConsumeOneTimeSecret("cust-r") // the original is spent (the dead-end premise)
+	if err != nil || pw1 == "" {
+		t.Fatal("harness: no initial secret")
+	}
+	if err := p.ReissueCredentials(context.Background(), "cust-r", "shared"); err != nil {
+		t.Fatalf("reissue: %v", err)
+	}
+	if fake.ResetCalls != 1 {
+		t.Fatalf("want exactly 1 sub-account password reset, got %d", fake.ResetCalls)
+	}
+	pw2, err := st.ConsumeOneTimeSecret("cust-r")
+	if err != nil || pw2 == "" {
+		t.Fatal("a FRESH one-time secret must be stored after reissue")
+	}
+	if pw2 == pw1 {
+		t.Fatal("the re-issued password must differ from the spent one")
+	}
+}
+
+// Scenario A (F4) — the dedicated path resets the labelled box's password.
+func TestReissue_DedicatedFreshSecret(t *testing.T) {
+	p, fake, st := newTestProvisioner(t)
+	if _, err := p.ProvisionOffsite(context.Background(), "cust-rd", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}); err != nil {
+		t.Fatal(err)
+	}
+	_, _ = st.ConsumeOneTimeSecret("cust-rd")
+	if err := p.ReissueCredentials(context.Background(), "cust-rd", "dedicated"); err != nil {
+		t.Fatalf("reissue dedicated: %v", err)
+	}
+	if fake.BoxResetCalls != 1 {
+		t.Fatalf("want exactly 1 box password reset, got %d", fake.BoxResetCalls)
+	}
+	if pw, err := st.ConsumeOneTimeSecret("cust-rd"); err != nil || pw == "" {
+		t.Fatal("fresh secret must be stored after a dedicated reissue")
+	}
+}
+
+// Scenario A WRONG-guard (F4) — an ambiguous label lookup (≠1 resource) must REFUSE: no reset, no secret.
+func TestReissue_RefusesAmbiguousLookup(t *testing.T) {
+	p, fake, st := newTestProvisioner(t)
+	// two sub-accounts labelled for the same customer (should never happen — refuse rather than guess)
+	fake.Subaccounts[1] = hetznerapi.Subaccount{ID: 1, StorageBox: 611421, Username: "u-sub1", Labels: map[string]string{"felhom-customer": "cust-amb"}}
+	fake.Subaccounts[2] = hetznerapi.Subaccount{ID: 2, StorageBox: 611421, Username: "u-sub2", Labels: map[string]string{"felhom-customer": "cust-amb"}}
+	err := p.ReissueCredentials(context.Background(), "cust-amb", "shared")
+	if err == nil || !strings.Contains(err.Error(), "exactly 1") {
+		t.Fatalf("ambiguous lookup must refuse, got %v", err)
+	}
+	if fake.ResetCalls != 0 {
+		t.Fatal("NO reset may run on an ambiguous lookup")
+	}
+	if _, cerr := st.ConsumeOneTimeSecret("cust-amb"); cerr == nil {
+		t.Fatal("NO secret may be stored on a refused reissue")
+	}
+	// zero resources → also refuse (nothing provisioned)
+	if err := p.ReissueCredentials(context.Background(), "cust-none", "shared"); err == nil {
+		t.Fatal("reissue with nothing provisioned must refuse")
+	}
+}
+
 // Scenario B — enable dedicated → box provisioned.
 func TestProvision_Dedicated(t *testing.T) {
 	p, fake, st := newTestProvisioner(t)