felhom.eu/hub/internal/offsite/offsite_test.go

package offsite

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"io"
	"log"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

func newTestProvisioner(t *testing.T) (*Provisioner, *hetznerapi.Fake, *store.Store) {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "off.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { st.Close() })
	fake := hetznerapi.NewFake()
	// ScanBackoff: empty (non-nil) → single scan attempt, no retries — tests that want the F2 retry set
	// their own schedule (nil would select the ~60s production default and stall the suite).
	return &Provisioner{API: fake, Store: st, Scanner: &fakeScanner{fp: "SHA256:testfp"}, PoolBoxID: 611421, Location: "fsn1", Logger: log.New(io.Discard, "", 0), ScanBackoff: []time.Duration{}}, fake, st
}

// flakyScanner fails the first `failures` calls (fresh-resource DNS lag), then succeeds.
type flakyScanner struct {
	failures int
	fp       string
	calls    int
}

func (f *flakyScanner) Fingerprint(_ context.Context, _ string, _ int) (string, error) {
	f.calls++
	if f.calls <= f.failures {
		return "", errors.New("dial tcp: lookup fresh.your-storagebox.de: no such host")
	}
	return f.fp, nil
}

// fakeScanner returns a fixed fingerprint (or an error) — no live SSH in tests.
type fakeScanner struct {
	fp  string
	err error
}

func (f *fakeScanner) Fingerprint(_ context.Context, _ string, _ int) (string, error) {
	return f.fp, f.err
}

// Scenario A — enable shared → sub-account provisioned, descriptor built, one-time password stored (NOT in
// ConfigJSON), password absent from the merged config.
func TestProvision_Shared(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	d, err := p.ProvisionOffsite(context.Background(), "cust-a", Input{Enabled: true, Type: "shared", QuotaGB: 50})
	if err != nil {
		t.Fatalf("provision: %v", err)
	}
	if fake.CreatedSubaccounts != 1 {
		t.Fatalf("want 1 subaccount created, got %d", fake.CreatedSubaccounts)
	}
	if d.Type != "shared" || d.Port != 23 || d.RepoPath != "/home/felhom-repo" || d.QuotaGB != 50 || d.User == "" || d.Host == "" {
		t.Fatalf("descriptor wrong: %+v", d)
	}
	// one-time password stored + is NOT the descriptor / config
	pw, err := st.ConsumeOneTimeSecret("cust-a")
	if err != nil || pw == "" {
		t.Fatalf("one-time password not stored: %v", err)
	}
	merged, err := MergeDescriptor(`{"git":{"token":"x"}}`, d)
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(merged, pw) {
		t.Fatal("the transient password LEAKED into ConfigJSON")
	}
	if !strings.Contains(merged, `"offsite"`) || !strings.Contains(merged, `"git"`) {
		t.Fatalf("merge lost keys: %s", merged)
	}
	// descriptor struct has no password field at all
	db, _ := json.Marshal(d)
	if strings.Contains(strings.ToLower(string(db)), "password") {
		t.Fatalf("descriptor carries a password field: %s", db)
	}
}

// Part 0 — the descriptor carries the box host-key fingerprint (captured at provision).
func TestProvision_HostFingerprint(t *testing.T) {
	p, _, _ := newTestProvisioner(t)
	d, err := p.ProvisionOffsite(context.Background(), "cust-fp", Input{Enabled: true, Type: "shared", QuotaGB: 10})
	if err != nil {
		t.Fatal(err)
	}
	if d.HostFingerprint != "SHA256:testfp" {
		t.Fatalf("descriptor must carry the host fingerprint, got %q", d.HostFingerprint)
	}
}

// Part 0 — a host-key scan failure is fail-closed (no descriptor served).
func TestProvision_ScanFailClosed(t *testing.T) {
	p, _, st := newTestProvisioner(t)
	p.Scanner = &fakeScanner{err: errors.New("keyscan timeout")}
	d, err := p.ProvisionOffsite(context.Background(), "cust-sf", Input{Enabled: true, Type: "shared", QuotaGB: 10})
	if err == nil || d != nil {
		t.Fatalf("a keyscan failure must fail-closed, got d=%+v err=%v", d, err)
	}
	// the resource may have been created + password stored, but no verifiable descriptor is served
	_ = st
}

// Scenario D (F2) — a fresh sub-account's DNS lags creation: the scan is retried and the FIRST save
// serves the descriptor.
func TestProvision_ScanRetriesThroughDNSLag(t *testing.T) {
	p, _, _ := newTestProvisioner(t)
	p.ScanBackoff = []time.Duration{0, 0, 0} // instant retries in tests
	sc := &flakyScanner{failures: 2, fp: "SHA256:late"}
	p.Scanner = sc
	d, err := p.ProvisionOffsite(context.Background(), "cust-dns", Input{Enabled: true, Type: "shared", QuotaGB: 10})
	if err != nil {
		t.Fatalf("the first save must survive DNS lag via scan retries, got: %v", err)
	}
	if d.HostFingerprint != "SHA256:late" || sc.calls != 3 {
		t.Fatalf("want fingerprint after 2 failed + 1 good scan, got fp=%q calls=%d", d.HostFingerprint, sc.calls)
	}
}

// Scenario D (F2) — a scan that keeps failing past the budget still fails CLOSED (no descriptor).
func TestProvision_ScanExhaustedFailsClosed(t *testing.T) {
	p, _, _ := newTestProvisioner(t)
	p.ScanBackoff = []time.Duration{0, 0}
	sc := &flakyScanner{failures: 99, fp: "SHA256:never"}
	p.Scanner = sc
	d, err := p.ProvisionOffsite(context.Background(), "cust-dns2", Input{Enabled: true, Type: "shared", QuotaGB: 10})
	if err == nil || d != nil {
		t.Fatalf("an exhausted scan budget must fail closed, got d=%+v err=%v", d, err)
	}
	if sc.calls != 3 { // 1 initial + 2 retries
		t.Fatalf("want 3 attempts (1 + 2 retries), got %d", sc.calls)
	}
}

// Scenario A (F4) — re-issue resets the labelled sub-account's password and stores a FRESH one-time
// secret (the consumed-password dead-end recovery).
func TestReissue_SharedFreshSecret(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	if _, err := p.ProvisionOffsite(context.Background(), "cust-r", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil {
		t.Fatal(err)
	}
	pw1, err := st.ConsumeOneTimeSecret("cust-r") // the original is spent (the dead-end premise)
	if err != nil || pw1 == "" {
		t.Fatal("harness: no initial secret")
	}
	if err := p.ReissueCredentials(context.Background(), "cust-r", "shared"); err != nil {
		t.Fatalf("reissue: %v", err)
	}
	if fake.ResetCalls != 1 {
		t.Fatalf("want exactly 1 sub-account password reset, got %d", fake.ResetCalls)
	}
	pw2, err := st.ConsumeOneTimeSecret("cust-r")
	if err != nil || pw2 == "" {
		t.Fatal("a FRESH one-time secret must be stored after reissue")
	}
	if pw2 == pw1 {
		t.Fatal("the re-issued password must differ from the spent one")
	}
}

// Scenario A (F4) — the dedicated path resets the labelled box's password.
func TestReissue_DedicatedFreshSecret(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	if _, err := p.ProvisionOffsite(context.Background(), "cust-rd", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}); err != nil {
		t.Fatal(err)
	}
	_, _ = st.ConsumeOneTimeSecret("cust-rd")
	if err := p.ReissueCredentials(context.Background(), "cust-rd", "dedicated"); err != nil {
		t.Fatalf("reissue dedicated: %v", err)
	}
	if fake.BoxResetCalls != 1 {
		t.Fatalf("want exactly 1 box password reset, got %d", fake.BoxResetCalls)
	}
	if pw, err := st.ConsumeOneTimeSecret("cust-rd"); err != nil || pw == "" {
		t.Fatal("fresh secret must be stored after a dedicated reissue")
	}
}

// Scenario A WRONG-guard (F4) — an ambiguous label lookup (≠1 resource) must REFUSE: no reset, no secret.
func TestReissue_RefusesAmbiguousLookup(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	// two sub-accounts labelled for the same customer (should never happen — refuse rather than guess)
	fake.Subaccounts[1] = hetznerapi.Subaccount{ID: 1, StorageBox: 611421, Username: "u-sub1", Labels: map[string]string{"felhom-customer": "cust-amb"}}
	fake.Subaccounts[2] = hetznerapi.Subaccount{ID: 2, StorageBox: 611421, Username: "u-sub2", Labels: map[string]string{"felhom-customer": "cust-amb"}}
	err := p.ReissueCredentials(context.Background(), "cust-amb", "shared")
	if err == nil || !strings.Contains(err.Error(), "exactly 1") {
		t.Fatalf("ambiguous lookup must refuse, got %v", err)
	}
	if fake.ResetCalls != 0 {
		t.Fatal("NO reset may run on an ambiguous lookup")
	}
	if _, cerr := st.ConsumeOneTimeSecret("cust-amb"); cerr == nil {
		t.Fatal("NO secret may be stored on a refused reissue")
	}
	// zero resources → also refuse (nothing provisioned)
	if err := p.ReissueCredentials(context.Background(), "cust-none", "shared"); err == nil {
		t.Fatal("reissue with nothing provisioned must refuse")
	}
}

// v0.57.0 (2.3, escrow honesty) — re-issuing offsite credentials INVALIDATES the key-escrow blob:
// the blob sealed the OLD repo password, so a recovery code minted against it would decrypt a
// password that no longer opens the repo. RED-PROOF (Scenario C): on pre-fix code (no MarkEscrowStale
// in ReissueCredentials + no stale plumbing) the hub keeps advertising the escrow as current after a
// re-issue and keeps serving its restic-hash for auto-confirm — this test asserts it does NEITHER.
func TestReissue_InvalidatesEscrow(t *testing.T) {
	p, _, st := newTestProvisioner(t)
	const cust = "cust-esc"
	if _, err := p.ProvisionOffsite(context.Background(), cust, Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil {
		t.Fatal(err)
	}
	// A host + a key-escrow blob whose sealed repo-password hash the hub serves for auto-confirm.
	if err := st.UpsertHost(&store.Host{HostID: cust + "-01", CustomerID: cust, APIKey: "k"}); err != nil {
		t.Fatal(err)
	}
	if _, _, err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob"), "SHA256:fp", "zero_knowledge", "2026-07-16T00:00:00Z", "OLDHASH"); err != nil {
		t.Fatal(err)
	}
	// Before re-issue: current escrow — the hub serves the sealed hash and is NOT stale.
	es, err := st.GetEscrowStatusForCustomer(cust)
	if err != nil || es == nil {
		t.Fatalf("escrow status (before): %v", err)
	}
	if es.Stale || es.ResticPwSHA256 != "OLDHASH" {
		t.Fatalf("pre-reissue escrow must be current: stale=%v hash=%q", es.Stale, es.ResticPwSHA256)
	}

	// Re-issue the offsite credential — the repo password just changed under the sealed blob.
	if err := p.ReissueCredentials(context.Background(), cust, "shared"); err != nil {
		t.Fatalf("reissue: %v", err)
	}

	// After: the escrow is STALE and the restic-hash is WITHHELD (no auto-confirm against a dead key).
	es, err = st.GetEscrowStatusForCustomer(cust)
	if err != nil || es == nil {
		t.Fatalf("escrow status (after): %v", err)
	}
	if !es.Stale {
		t.Fatal("RED-PROOF: escrow must be STALE after an offsite re-issue (the hub was advertising ceremony-done against a key the repo no longer accepts)")
	}
	if es.ResticPwSHA256 != "" {
		t.Fatalf("a stale escrow must WITHHOLD the restic hash to inhibit auto-confirm, got %q", es.ResticPwSHA256)
	}

	// A fresh ceremony (new blob sealing the new password) clears stale + serves the new hash.
	if _, _, err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob-2"), "SHA256:fp", "zero_knowledge", "2026-07-16T01:00:00Z", "NEWHASH"); err != nil {
		t.Fatal(err)
	}
	es, _ = st.GetEscrowStatusForCustomer(cust)
	if es == nil || es.Stale || es.ResticPwSHA256 != "NEWHASH" {
		t.Fatalf("a fresh ceremony must clear stale + serve the new hash: %+v", es)
	}
}

// Scenario E (SLICE 4) — the freeze lever flips ONLY readonly on the exactly-1 labelled sub-account
// (SSH stays on — a freeze must not cut access, just writes); ambiguity refuses; unfreeze reverses.
func TestFreeze_SharedTogglesReadonlyOnly(t *testing.T) {
	p, fake, _ := newTestProvisioner(t)
	if _, err := p.ProvisionOffsite(context.Background(), "cust-fz", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil {
		t.Fatal(err)
	}
	if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", true); err != nil {
		t.Fatalf("freeze: %v", err)
	}
	subs, _ := fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz")
	if len(subs) != 1 || !subs[0].AccessSettings.Readonly {
		t.Fatalf("freeze must set readonly on the labelled sub-account: %+v", subs)
	}
	if !subs[0].AccessSettings.SSHEnabled {
		t.Fatal("freeze must NOT disable SSH — only readonly flips")
	}
	if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", false); err != nil {
		t.Fatalf("unfreeze: %v", err)
	}
	subs, _ = fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz")
	if subs[0].AccessSettings.Readonly {
		t.Fatal("unfreeze must clear readonly")
	}
	// nothing provisioned → refuse
	if err := p.SetOffsiteFrozen(context.Background(), "cust-none", true); err == nil {
		t.Fatal("freeze with no labelled sub-account must refuse")
	}
	// ambiguous → refuse
	fake.Subaccounts[71] = hetznerapi.Subaccount{ID: 71, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}}
	fake.Subaccounts[72] = hetznerapi.Subaccount{ID: 72, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}}
	if err := p.SetOffsiteFrozen(context.Background(), "cust-amb2", true); err == nil || !strings.Contains(err.Error(), "exactly 1") {
		t.Fatalf("ambiguous freeze must refuse, got %v", err)
	}
}

// Scenario B — enable dedicated → box provisioned.
func TestProvision_Dedicated(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	d, err := p.ProvisionOffsite(context.Background(), "cust-b", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"})
	if err != nil {
		t.Fatalf("provision: %v", err)
	}
	if fake.CreatedBoxes != 1 {
		t.Fatalf("want 1 box created, got %d", fake.CreatedBoxes)
	}
	if d.Type != "dedicated" || d.BoxType != "bx11" || d.User == "" || d.Host == "" || d.RepoPath != "/home/felhom-repo" {
		t.Fatalf("descriptor wrong: %+v", d)
	}
	if pw, err := st.ConsumeOneTimeSecret("cust-b"); err != nil || pw == "" {
		t.Fatalf("one-time password not stored: %v", err)
	}
}

// Scenario C — idempotent re-save does NOT create a second resource.
func TestProvision_Idempotent(t *testing.T) {
	p, fake, _ := newTestProvisioner(t)
	if _, err := p.ProvisionOffsite(context.Background(), "cust-c", Input{Enabled: true, Type: "shared", QuotaGB: 20}); err != nil {
		t.Fatal(err)
	}
	d2, err := p.ProvisionOffsite(context.Background(), "cust-c", Input{Enabled: true, Type: "shared", QuotaGB: 20})
	if err != nil {
		t.Fatal(err)
	}
	if fake.CreatedSubaccounts != 1 {
		t.Fatalf("re-provision created a SECOND resource (%d) — not idempotent", fake.CreatedSubaccounts)
	}
	if d2.User == "" || d2.Type != "shared" {
		t.Fatalf("idempotent descriptor wrong: %+v", d2)
	}
}

// Scenario D — a provisioning error surfaces; nothing is recorded (fail-closed). Companion: the caller must
// not mark offsite provisioned — modelled here by asserting no descriptor + no stored password on error.
func TestProvision_FailClosed(t *testing.T) {
	p, fake, st := newTestProvisioner(t)
	fake.FailCreate = errors.New("hetzner 500")
	d, err := p.ProvisionOffsite(context.Background(), "cust-d", Input{Enabled: true, Type: "shared", QuotaGB: 10})
	if err == nil {
		t.Fatal("a create failure must return an error (fail-closed)")
	}
	if d != nil {
		t.Fatalf("no descriptor may be returned on error, got %+v", d)
	}
	if _, cerr := st.ConsumeOneTimeSecret("cust-d"); cerr != sql.ErrNoRows {
		t.Fatal("no one-time password may be stored on a failed provision")
	}
	// action-failure path (create ok, action errors) is also fail-closed
	fake.FailCreate = nil
	fake.FailAction = true
	if _, err := p.ProvisionOffsite(context.Background(), "cust-d2", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}); err == nil {
		t.Fatal("a failed create-action must return an error")
	}
}

// Scenario E — the one-time password is consumable exactly once.
func TestOneTimeSecret_ConsumedOnce(t *testing.T) {
	_, _, st := newTestProvisioner(t)
	if err := st.SaveOneTimeSecret("cust-e", "secretpw"); err != nil {
		t.Fatal(err)
	}
	got, err := st.ConsumeOneTimeSecret("cust-e")
	if err != nil || got != "secretpw" {
		t.Fatalf("first consume: got %q err %v", got, err)
	}
	if _, err := st.ConsumeOneTimeSecret("cust-e"); err != sql.ErrNoRows {
		t.Fatalf("second consume must be ErrNoRows, got %v", err)
	}
	if _, err := st.ConsumeOneTimeSecret("never-provisioned"); err != sql.ErrNoRows {
		t.Fatalf("absent consume must be ErrNoRows, got %v", err)
	}
}

// Disable → {Enabled:false}, no deprovision (no API delete).
func TestProvision_DisableNoDeprovision(t *testing.T) {
	p, fake, _ := newTestProvisioner(t)
	d, err := p.ProvisionOffsite(context.Background(), "cust-f", Input{Enabled: false})
	if err != nil || d == nil || d.Enabled {
		t.Fatalf("disable must return {enabled:false}, got %+v err %v", d, err)
	}
	if fake.DeletedSubaccounts != 0 || fake.DeletedBoxes != 0 {
		t.Fatal("disable must NOT deprovision (data-loss guard)")
	}
}

// Customer RESET (v0.61.0) — Deprovision DESTROYS the labelled shared sub-account, and is idempotent
// (a second call finds nothing and succeeds). This is the deliberate teardown the disable-guard above
// deliberately does NOT do.
func TestDeprovision_SharedIdempotent(t *testing.T) {
	p, fake, _ := newTestProvisioner(t)
	if _, err := p.ProvisionOffsite(context.Background(), "cust-d", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil {
		t.Fatalf("provision: %v", err)
	}
	if fake.CreatedSubaccounts != 1 {
		t.Fatalf("precondition: want 1 subaccount, got %d", fake.CreatedSubaccounts)
	}
	if err := p.Deprovision(context.Background(), "cust-d", "shared"); err != nil {
		t.Fatalf("deprovision: %v", err)
	}
	if fake.DeletedSubaccounts != 1 {
		t.Fatalf("want 1 subaccount deleted, got %d", fake.DeletedSubaccounts)
	}
	// Idempotent: nothing labelled now → success, no extra delete.
	if err := p.Deprovision(context.Background(), "cust-d", "shared"); err != nil {
		t.Fatalf("second deprovision (idempotent) errored: %v", err)
	}
	if fake.DeletedSubaccounts != 1 {
		t.Fatalf("idempotent re-run deleted again: %d", fake.DeletedSubaccounts)
	}
}

// ClearProvisionedDescriptor keeps the tier CHOICE (enabled/type/quota/box_type) and drops every
// PROVISIONED field — the pre-first-install shape a RESET returns the customer to.
func TestClearProvisionedDescriptor(t *testing.T) {
	in := `{"git":{"token":"x"},"offsite":{"enabled":true,"type":"shared","host":"u1-sub3.your-storagebox.de","user":"u1-sub3","port":23,"repo_path":"/home/felhom","quota_gb":100,"box_type":"","host_fingerprint":"SHA256:abc"}}`
	out, err := ClearProvisionedDescriptor(in)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out, `"enabled":true`) || !strings.Contains(out, `"type":"shared"`) || !strings.Contains(out, `"quota_gb":100`) {
		t.Errorf("tier choice lost: %s", out)
	}
	for _, gone := range []string{"your-storagebox.de", "u1-sub3", "repo_path", "host_fingerprint", `"port"`} {
		if strings.Contains(out, gone) {
			t.Errorf("provisioned field %q survived: %s", gone, out)
		}
	}
	if !strings.Contains(out, `"git"`) {
		t.Errorf("unrelated config keys dropped: %s", out)
	}
	// No-op-safe: absent offsite block returns input unchanged.
	if got, _ := ClearProvisionedDescriptor(`{"git":{"token":"x"}}`); got != `{"git":{"token":"x"}}` {
		t.Errorf("no-offsite clear mutated config: %s", got)
	}
}