// Package offsite orchestrates per-customer offsite-tier provisioning against the Hetzner storage-box API // (SLICE 1): idempotent create of a shared sub-account or a dedicated box, generation of the transient // one-time password, and the NON-SECRET target descriptor that rides ConfigJSON to the controller. The // controller-side apply-bridge (SLICE 2) and escrow auto-confirm (SLICE 3) are out of scope here. // // Fail-closed: any API/action error returns without a provisioned resource being recorded — the caller // must NOT mark offsite enabled/served on error. Idempotent: every create is guarded by a label lookup // first (box/sub-account names are not unique — SPIKE §2). package offsite import ( "context" "crypto/rand" "encoding/json" "fmt" "log" "math/big" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // sftpPort is the storage-box SSH/SFTP port (SPIKE: 23). const sftpPort = 23 // Descriptor is the NON-SECRET offsite target that rides ConfigJSON to the controller. It NEVER carries // the password or the SSH key. type Descriptor struct { Enabled bool `json:"enabled"` Type string `json:"type,omitempty"` // "shared" | "dedicated" Host string `json:"host,omitempty"` // .your-storagebox.de (dedicated) / -subN… (shared) User string `json:"user,omitempty"` Port int `json:"port,omitempty"` // 23 RepoPath string `json:"repo_path,omitempty"` // /home/ QuotaGB int `json:"quota_gb,omitempty"` // shared soft-quota (Felhom-enforced; no native lever) BoxType string `json:"box_type,omitempty"` // dedicated (Hetzner-hard quota via the type) // HostFingerprint is the box's SSH host-key fingerprint (SHA256:…), captured at provision so the // controller VERIFIES the box identity instead of blind-TOFU (SLICE 2). Non-secret. HostFingerprint string `json:"host_fingerprint,omitempty"` } // HostKeyScanner returns a box's SSH host-key fingerprint (SHA256:…). Seam'd so tests inject a fake. type HostKeyScanner interface { Fingerprint(ctx context.Context, host string, port int) (string, error) } // Input is the operator's offsite choice. type Input struct { Enabled bool Type string // "shared" | "dedicated" QuotaGB int // shared BoxType string // dedicated, e.g. "bx11" } // Provisioner provisions offsite resources. It depends on the CloudAPI interface (tests inject a fake). type Provisioner struct { API hetznerapi.CloudAPI Store *store.Store Scanner HostKeyScanner // captures the box host-key fingerprint (fail-closed if nil/scan-fails) PoolBoxID int64 // the shared-pool storage-box id (e.g. 611421) Location string // dedicated-box location, e.g. "fsn1" Logger *log.Logger // ScanBackoff is the retry schedule for the host-key scan (F2: a fresh sub-account's DNS name lags // creation by seconds-to-a-minute, so the first scan typically fails with "no such host"). nil → the // default ~60s ladder. Tests inject zeros. The total must fit inside applyOffsite's 3-min detached ctx. ScanBackoff []time.Duration } // defaultScanBackoff: 5 retries, ~60s total — sized to the observed DNS propagation lag. var defaultScanBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second} func (p *Provisioner) logf(f string, a ...any) { if p.Logger != nil { p.Logger.Printf(f, a...) } } // customerLabel is the idempotency/teardown key. func customerLabel(customerID string) map[string]string { return map[string]string{"felhom-customer": customerID} } func customerSelector(customerID string) string { return "felhom-customer=" + customerID } // repoPath is the controller-facing RepoPath — each account is chrooted, /home is writable (SPIKE). const repoPath = "/home/felhom-repo" // ProvisionOffsite ensures the customer's offsite resource exists and returns the non-secret descriptor. // On a fresh create it generates + stores the one-time password (Store.SaveOneTimeSecret). On an existing // resource (found by label) it is a no-op create → returns the descriptor without a new password. The // caller merges the descriptor into ConfigJSON and saves. Disable → returns {Enabled:false} (NO deprovision). func (p *Provisioner) ProvisionOffsite(ctx context.Context, customerID string, in Input) (*Descriptor, error) { if !in.Enabled { return &Descriptor{Enabled: false}, nil } var d *Descriptor var err error switch in.Type { case "shared": d, err = p.provisionShared(ctx, customerID, in) case "dedicated": d, err = p.provisionDedicated(ctx, customerID, in) default: return nil, fmt.Errorf("offsite: unknown type %q (want shared|dedicated)", in.Type) } if err != nil { return nil, err } // Capture the box host-key fingerprint so the controller verifies (no blind TOFU). Fail-closed: don't // serve a descriptor the controller can't verify. Applies to both fresh and idempotent paths. if p.Scanner == nil { return nil, fmt.Errorf("offsite: no host-key scanner configured (cannot capture the pin)") } port := d.Port if port == 0 { port = sftpPort } fp, err := p.scanWithRetry(ctx, d.Host, port) if err != nil { return nil, fmt.Errorf("offsite: host-key scan %s: %w", d.Host, err) } d.HostFingerprint = fp return d, nil } // scanWithRetry retries the host-key scan on failure (F2: fresh-resource DNS lag). Fail-closed past the // budget; ctx cancellation aborts between attempts. func (p *Provisioner) scanWithRetry(ctx context.Context, host string, port int) (string, error) { backoff := p.ScanBackoff if backoff == nil { backoff = defaultScanBackoff } fp, err := p.Scanner.Fingerprint(ctx, host, port) for i := 0; err != nil && i < len(backoff); i++ { p.logf("[offsite] host-key scan %s failed (attempt %d/%d, retrying in %s): %v", host, i+1, len(backoff)+1, backoff[i], err) select { case <-ctx.Done(): return "", ctx.Err() case <-time.After(backoff[i]): } fp, err = p.Scanner.Fingerprint(ctx, host, port) } return fp, err } // ReissueCredentials resets the customer's offsite credential and stores a FRESH one-time password — the // EXPLICIT operator recovery for a consumed-password dead-end (a fresh guest at DR, or a // consumed-but-failed install). It is NOT implicit rotation: ProvisionOffsite never calls this. Scoped // hard: the reset targets ONLY the resource labelled `felhom-customer=`, and refuses unless the label // lookup finds exactly one. The password value is never logged (the action is). func (p *Provisioner) ReissueCredentials(ctx context.Context, customerID, typ string) error { pw, err := genPassword() if err != nil { return err } switch typ { case "shared": if p.PoolBoxID == 0 { return fmt.Errorf("offsite: no shared pool box configured") } subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) if err != nil { return fmt.Errorf("offsite: reissue lookup: %w", err) } if len(subs) != 1 { return fmt.Errorf("offsite: reissue needs exactly 1 sub-account labelled for %s, found %d — refusing", customerID, len(subs)) } act, err := p.API.ResetSubaccountPassword(ctx, p.PoolBoxID, subs[0].ID, pw) if err != nil { return fmt.Errorf("offsite: reset sub-account password: %w", err) } if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: reset action: %w", err) } p.logf("[offsite] re-issued shared credentials for %s (subaccount %d)", customerID, subs[0].ID) case "dedicated": boxes, err := p.API.ListStorageBoxes(ctx, customerSelector(customerID)) if err != nil { return fmt.Errorf("offsite: reissue lookup: %w", err) } if len(boxes) != 1 { return fmt.Errorf("offsite: reissue needs exactly 1 box labelled for %s, found %d — refusing", customerID, len(boxes)) } act, err := p.API.ResetBoxPassword(ctx, boxes[0].ID, pw) if err != nil { return fmt.Errorf("offsite: reset box password: %w", err) } if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: reset action: %w", err) } p.logf("[offsite] re-issued dedicated credentials for %s (box %d)", customerID, boxes[0].ID) default: return fmt.Errorf("offsite: reissue: unknown type %q (want shared|dedicated)", typ) } if err := p.Store.SaveOneTimeSecret(customerID, pw); err != nil { return fmt.Errorf("offsite: store re-issued one-time password: %w", err) } // v0.57.0 (2.3, the escrow-honesty fix): the restic repo password just changed, so any existing // key-escrow blob — which sealed the OLD password — is now STALE. A recovery code minted against // it would decrypt a password that no longer opens the repo. Mark the escrow stale so the hub // stops advertising "ceremony done" and the customer's escrow wizard is offered again; a fresh // ceremony seals the new password and clears the flag. Every credential change also emits a // visible customer event (offsite_reissued always; escrow_stale only when a blob was invalidated). // Best-effort: the password reset already succeeded — a bookkeeping failure here must not fail it. escrowStaled := false if host, herr := p.Store.GetHostByCustomer(customerID); herr == nil && host != nil { if esc, eerr := p.Store.GetHostEscrow(host.HostID); eerr == nil && esc != nil { if serr := p.Store.MarkEscrowStale(host.HostID); serr != nil { p.logf("[offsite] WARN mark-escrow-stale for %s: %v", customerID, serr) } else { escrowStaled = true } } } if _, serr := p.Store.SaveEvent(customerID, "offsite_reissued", "info", "Az offsite (házon kívüli) mentési hozzáférést újra kiadtuk — az új egyszeri jelszót a vezérlő a következő frissítéskor átveszi.", "", "hub"); serr != nil { p.logf("[offsite] WARN save offsite_reissued event for %s: %v", customerID, serr) } if escrowStaled { if _, serr := p.Store.SaveEvent(customerID, "escrow_stale", "warning", "A helyreállítási kulcs-letét elavult az offsite jelszó cseréje miatt — futtasd le újra a helyreállítási szertartást (Biztonsági mentés → Helyreállítás).", "", "hub"); serr != nil { p.logf("[offsite] WARN save escrow_stale event for %s: %v", customerID, serr) } } return nil } // OffsiteIdentifier returns the customer's provisioned offsite name (sub-account username / box name) // for the RESET preview inventory, "" if none is provisioned. Read-only. func (p *Provisioner) OffsiteIdentifier(ctx context.Context, customerID, typ string) (string, error) { switch typ { case "dedicated": boxes, err := p.API.ListStorageBoxes(ctx, customerSelector(customerID)) if err != nil || len(boxes) == 0 { return "", err } return boxes[0].Name, nil default: // shared / "" if p.PoolBoxID == 0 { return "", nil } subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) if err != nil || len(subs) == 0 { return "", err } return subs[0].Username, nil } } // Deprovision DELETES the customer's offsite storage — the RESET teardown (v0.61.0). The offsite repo // DATA dies with the sub-account/box; irreversible, gated by the operator RESET confirm. IDEMPOTENT: // zero labelled sub-accounts/boxes = already gone = success (a re-run after a partial reset does not // error). The id is re-derived by the customer label each time — nothing stored to go stale. func (p *Provisioner) Deprovision(ctx context.Context, customerID, typ string) error { switch typ { case "dedicated": boxes, err := p.API.ListStorageBoxes(ctx, customerSelector(customerID)) if err != nil { return fmt.Errorf("offsite: deprovision lookup: %w", err) } if len(boxes) == 0 { p.logf("[offsite] deprovision: no box labelled for %s — already gone", customerID) return nil } for _, b := range boxes { act, err := p.API.DeleteStorageBox(ctx, b.ID) if err != nil { return fmt.Errorf("offsite: delete box %d: %w", b.ID, err) } if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: delete box action: %w", err) } p.logf("[offsite] deprovisioned dedicated box %d for %s (repo data destroyed)", b.ID, customerID) } return nil default: // shared / "" if p.PoolBoxID == 0 { return fmt.Errorf("offsite: no shared pool box configured") } subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) if err != nil { return fmt.Errorf("offsite: deprovision lookup: %w", err) } if len(subs) == 0 { p.logf("[offsite] deprovision: no sub-account labelled for %s — already gone", customerID) return nil } for _, sub := range subs { act, err := p.API.DeleteSubaccount(ctx, p.PoolBoxID, sub.ID) if err != nil { return fmt.Errorf("offsite: delete sub-account %d: %w", sub.ID, err) } if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: delete sub-account action: %w", err) } p.logf("[offsite] deprovisioned shared sub-account %d for %s (repo data destroyed)", sub.ID, customerID) } return nil } } func (p *Provisioner) provisionShared(ctx context.Context, customerID string, in Input) (*Descriptor, error) { if p.PoolBoxID == 0 { return nil, fmt.Errorf("offsite: no shared pool box configured") } // Idempotency: an existing labelled sub-account is reused (no second create). existing, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) if err != nil { return nil, fmt.Errorf("offsite: list subaccounts: %w", err) } if len(existing) > 0 { s := existing[0] p.logf("[offsite] shared already provisioned for %s (subaccount %d)", customerID, s.ID) return &Descriptor{Enabled: true, Type: "shared", Host: s.Server, User: s.Username, Port: sftpPort, RepoPath: repoPath, QuotaGB: in.QuotaGB}, nil } // Fresh create: generate the transient password, create, wait, fetch the full object. pw, err := genPassword() if err != nil { return nil, err } id, action, err := p.API.CreateSubaccount(ctx, p.PoolBoxID, hetznerapi.CreateSubaccountRequest{ HomeDirectory: "felhom-" + customerID, Password: pw, AccessSettings: hetznerapi.AccessSettings{SSHEnabled: true, ReachableExternally: true}, Labels: customerLabel(customerID), Description: "felhom offsite " + customerID, }) if err != nil { return nil, fmt.Errorf("offsite: create subaccount: %w", err) } if err := p.API.WaitAction(ctx, action); err != nil { return nil, fmt.Errorf("offsite: subaccount create action: %w", err) } sub, err := p.API.GetSubaccount(ctx, p.PoolBoxID, id) if err != nil { return nil, fmt.Errorf("offsite: fetch subaccount: %w", err) } if err := p.Store.SaveOneTimeSecret(customerID, pw); err != nil { return nil, fmt.Errorf("offsite: store one-time password: %w", err) } p.logf("[offsite] shared provisioned for %s (subaccount %d, user %s)", customerID, sub.ID, sub.Username) return &Descriptor{Enabled: true, Type: "shared", Host: sub.Server, User: sub.Username, Port: sftpPort, RepoPath: repoPath, QuotaGB: in.QuotaGB}, nil } func (p *Provisioner) provisionDedicated(ctx context.Context, customerID string, in Input) (*Descriptor, error) { boxType := in.BoxType if boxType == "" { boxType = "bx11" } existing, err := p.API.ListStorageBoxes(ctx, customerSelector(customerID)) if err != nil { return nil, fmt.Errorf("offsite: list boxes: %w", err) } if len(existing) > 0 { b := existing[0] p.logf("[offsite] dedicated already provisioned for %s (box %d)", customerID, b.ID) return &Descriptor{Enabled: true, Type: "dedicated", Host: b.Server, User: b.Username, Port: sftpPort, RepoPath: repoPath, BoxType: boxType}, nil } pw, err := genPassword() if err != nil { return nil, err } id, action, err := p.API.CreateStorageBox(ctx, hetznerapi.CreateBoxRequest{ Name: "felhom-" + customerID, StorageBoxType: boxType, Location: p.Location, Password: pw, AccessSettings: hetznerapi.AccessSettings{SSHEnabled: true, ReachableExternally: true}, Labels: customerLabel(customerID), }) if err != nil { return nil, fmt.Errorf("offsite: create box: %w", err) } if err := p.API.WaitAction(ctx, action); err != nil { return nil, fmt.Errorf("offsite: box create action: %w", err) } box, err := p.API.GetStorageBox(ctx, id) if err != nil { return nil, fmt.Errorf("offsite: fetch box: %w", err) } if err := p.Store.SaveOneTimeSecret(customerID, pw); err != nil { return nil, fmt.Errorf("offsite: store one-time password: %w", err) } p.logf("[offsite] dedicated provisioned for %s (box %d, user %s)", customerID, box.ID, box.Username) return &Descriptor{Enabled: true, Type: "dedicated", Host: box.Server, User: box.Username, Port: sftpPort, RepoPath: repoPath, BoxType: boxType}, nil } // SetOffsiteFrozen freezes/unfreezes the customer's SHARED sub-account (SLICE 4: readonly access) — an // OPERATOR lever, NEVER automatic: freezing also blocks prune/forget, which is the customer's only way // DOWN from over-quota, so only a human weighs that trade-off. Same exactly-1 label guard as the // re-issue. Preserves the sub-account's other access settings (SSH must stay on — only readonly flips). // Dedicated boxes have no freeze path (Hetzner enforces their size physically; the UI hides the button). func (p *Provisioner) SetOffsiteFrozen(ctx context.Context, customerID string, frozen bool) error { if p.PoolBoxID == 0 { return fmt.Errorf("offsite: no shared pool box configured") } subs, err := p.API.ListSubaccounts(ctx, p.PoolBoxID, customerSelector(customerID)) if err != nil { return fmt.Errorf("offsite: freeze lookup: %w", err) } if len(subs) != 1 { return fmt.Errorf("offsite: freeze needs exactly 1 sub-account labelled for %s, found %d — refusing", customerID, len(subs)) } as := subs[0].AccessSettings as.Readonly = frozen act, err := p.API.UpdateSubaccountAccess(ctx, p.PoolBoxID, subs[0].ID, as) if err != nil { return fmt.Errorf("offsite: update access: %w", err) } if err := p.API.WaitAction(ctx, act); err != nil { return fmt.Errorf("offsite: freeze action: %w", err) } p.logf("[offsite] set frozen=%v (readonly) for %s (subaccount %d)", frozen, customerID, subs[0].ID) return nil } // MergeDescriptor merges the offsite descriptor under the "offsite" key of a ConfigJSON object, preserving // all other keys. Returns the new ConfigJSON string. NEVER carries a secret (Descriptor is non-secret). // ClearProvisionedDescriptor returns config_json with the offsite descriptor reset to its // pre-first-install shape — the customer-RESET clear (v0.61.0). The TIER CHOICE // (enabled/type/quota_gb/box_type — customer config, SURVIVES a RESET) is kept; every PROVISIONED // field (host/user/port/repo_path/host_fingerprint — operational state, DIES with the Hetzner // resource) is cleared, so re-onboarding re-provisions fresh against the retained choice. No-op-safe: // an absent/empty offsite block returns the input unchanged. func ClearProvisionedDescriptor(configJSON string) (string, error) { cur, err := ReadDescriptor(configJSON) if err != nil { return "", err } if cur == nil { return configJSON, nil // no offsite block — nothing to clear } cleared := &Descriptor{Enabled: cur.Enabled, Type: cur.Type, QuotaGB: cur.QuotaGB, BoxType: cur.BoxType} return MergeDescriptor(configJSON, cleared) } // ReadDescriptor extracts the non-secret offsite Descriptor from a customer's ConfigJSON, or (nil, nil) // when there is no offsite block (never provisioned). This is the AUTHORITATIVE tier/quota source — the // pool-box Σ(quota) aggregate (v0.64.0, R-5) and the RESET clear both read through it, NEVER the report // echo (a stale/absent report would undercount the sold promises; ConfigJSON is the operator's intent). func ReadDescriptor(configJSON string) (*Descriptor, error) { obj := map[string]json.RawMessage{} if strings.TrimSpace(configJSON) != "" && configJSON != "{}" { if err := json.Unmarshal([]byte(configJSON), &obj); err != nil { return nil, fmt.Errorf("offsite: parse config_json: %w", err) } } raw, ok := obj["offsite"] if !ok { return nil, nil } var d Descriptor if err := json.Unmarshal(raw, &d); err != nil { return nil, fmt.Errorf("offsite: parse offsite descriptor: %w", err) } return &d, nil } func MergeDescriptor(configJSON string, d *Descriptor) (string, error) { obj := map[string]json.RawMessage{} if strings.TrimSpace(configJSON) != "" && configJSON != "{}" { if err := json.Unmarshal([]byte(configJSON), &obj); err != nil { return "", fmt.Errorf("offsite: parse config_json: %w", err) } } db, err := json.Marshal(d) if err != nil { return "", err } obj["offsite"] = db out, err := json.Marshal(obj) if err != nil { return "", err } return string(out), nil } // genPassword returns a transient password satisfying the Hetzner 4-class policy (upper+lower+digit+special). func genPassword() (string, error) { const alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" b := make([]byte, 24) for i := range b { n, err := rand.Int(rand.Reader, big.NewInt(int64(len(alnum)))) if err != nil { return "", err } b[i] = alnum[n.Int64()] } // Guarantee all four classes (transient + single-use + reset after install). return string(b) + "Aa9%", nil }