368 lines
14 KiB
YAML
368 lines
14 KiB
YAML
# Felhom Hub — Multi-customer dashboard
|
||
# Dashboard: https://hub.felhom.eu
|
||
# API: POST /api/v1/report (Bearer token auth)
|
||
#
|
||
# Receives health reports from customer controllers and displays
|
||
# a centralized overview dashboard for the operator (Viktor).
|
||
#
|
||
# Namespace: felhom-system (shared with healthchecks and other felhom infra)
|
||
#
|
||
# PREREQUISITES:
|
||
# 1. Build and push the hub image:
|
||
# cd ~/build/felhom-hub && ./build.sh v0.2.0 --push
|
||
#
|
||
# 2. Generate a bcrypt password hash for dashboard login:
|
||
# htpasswd -nbBC 10 "" "your-password" | cut -d: -f2
|
||
# Update the ConfigMap password_hash field below.
|
||
#
|
||
# 3. Create the operator/global bearer key Secret (out-of-band, NEVER committed):
|
||
# openssl rand -hex 32 # mint
|
||
# kubectl -n felhom-system create secret generic report-api \
|
||
# --from-literal=REPORT_API_KEY=<minted-key>
|
||
# (Customer boxes use per-customer/per-host keys generated by the hub — the global
|
||
# key is the operator's own, e.g. felhom-ops -hub-key.)
|
||
#
|
||
# 4. Apply this manifest:
|
||
# kubectl apply -f manifests/hub.yaml
|
||
#
|
||
# 5. Configure DNS:
|
||
# Add hub.felhom.eu → k3s cluster IP in Cloudflare
|
||
#
|
||
# DEBUGGING:
|
||
# kubectl logs -n felhom-system deploy/hub -f
|
||
# kubectl exec -it -n felhom-system deploy/hub -- ls /data/
|
||
# kubectl describe ingress -n felhom-system hub
|
||
|
||
# =============================================================================
|
||
# PERSISTENT STORAGE
|
||
# =============================================================================
|
||
---
|
||
apiVersion: v1
|
||
kind: PersistentVolumeClaim
|
||
metadata:
|
||
name: hub-data
|
||
namespace: felhom-system
|
||
labels:
|
||
app: hub
|
||
# D40 (operator 2026-10-24, R-1243): the hub's volume is NOT in DooPlex's Longhorn backup — that copy was plain
|
||
# and sat in an open NFS folder. The hub's off-site copy is the encrypted 6-hourly DB copy to ep0
|
||
# (scripts/hub-db-backup, `05` §16.2). `disabled`, not a removed label: ArgoCD does not prune a label, and a
|
||
# volume with no recurring-job label falls into Longhorn's default group (backed up).
|
||
recurring-job-group.longhorn.io/default: disabled
|
||
spec:
|
||
accessModes:
|
||
- ReadWriteOnce
|
||
storageClassName: longhorn
|
||
resources:
|
||
requests:
|
||
storage: 4Gi
|
||
|
||
# =============================================================================
|
||
# CONFIGURATION
|
||
# =============================================================================
|
||
---
|
||
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: hub-config
|
||
namespace: felhom-system
|
||
data:
|
||
hub.yaml: |
|
||
auth:
|
||
# Bcrypt hash for dashboard login (Viktor only)
|
||
# Generate: htpasswd -nbBC 10 "" "your-password" | cut -d: -f2
|
||
password_hash: "$2y$10$N5.O9jBnc.1tIlJT/irx3OlVjJQemlCHRnfqIJg/EyZofnzXSCpeG"
|
||
api:
|
||
# Operator/global bearer key. NOT stored here since v0.53.0 — injected at runtime from
|
||
# Secret/report-api via the REPORT_API_KEY env var (see Deployment below). The Secret is
|
||
# created out-of-band and NOT committed (documentation/runbooks/secrets.md); the previously
|
||
# committed literal is retired by ROTATION (see the publish-runbook notes). Leave empty.
|
||
report_api_key: ""
|
||
retention:
|
||
max_days: 90
|
||
prune_schedule: "04:30"
|
||
alerting:
|
||
# 45m: operator ruling A on R-549 (2026-09-17) - three report cycles, so one failed push
|
||
# (retried ~100 s, measured 29m59s gap in chaos night round 9) cannot page on a healthy box.
|
||
# node_down / host_down follow at 2x = 90m.
|
||
stale_threshold: "45m"
|
||
notifications:
|
||
# Resend API key is NOT stored here. It is injected at runtime from Secret/resend-api
|
||
# via the RESEND_API_KEY env var (see Deployment below). The Secret is created out-of-band
|
||
# and is NOT committed — see documentation/runbooks/secrets.md. Leave this empty.
|
||
resend_api_key: ""
|
||
# Operator alert recipient + enable. WITHOUT both, Dispatcher.processOperator returns early and
|
||
# NO operator email is ever sent — the self-health pipeline (probe→report→checker→dispatch) stops
|
||
# one hop short of the inbox (TESTRUN finding: the unproven hop). The address is the operator's own
|
||
# and is not a secret. from_email defaults to monitoring@felhom.eu.
|
||
operator_email: "admin@felhom.eu"
|
||
operator_enabled: true
|
||
registry:
|
||
image: "gitea.dooplex.hu/admin/felhom-controller"
|
||
# username + token injected via REGISTRY_USERNAME / REGISTRY_TOKEN env vars
|
||
# from Secret/gitea-creds (see Deployment below)
|
||
check_interval: "6h"
|
||
template_interval: "1h"
|
||
server:
|
||
listen: ":8080"
|
||
data_dir: "/data"
|
||
|
||
# =============================================================================
|
||
# DEPLOYMENT
|
||
# =============================================================================
|
||
---
|
||
apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: hub
|
||
namespace: felhom-system
|
||
labels:
|
||
app: hub
|
||
spec:
|
||
replicas: 1
|
||
strategy:
|
||
type: Recreate
|
||
selector:
|
||
matchLabels:
|
||
app: hub
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: hub
|
||
spec:
|
||
containers:
|
||
- name: hub
|
||
image: gitea.dooplex.hu/admin/felhom-hub:0.174.1
|
||
ports:
|
||
- containerPort: 8080
|
||
name: http
|
||
env:
|
||
- name: TZ
|
||
value: "Europe/Budapest"
|
||
# Phase 2 managed updates: global controller-version FLOOR fallback. Any reporting box below
|
||
# this auto-updates to it (unless a per-customer override is set via the operator UI).
|
||
- name: DEFAULT_MIN_CONTROLLER_VERSION
|
||
value: "0.120.0"
|
||
# Resend API key — injected from the out-of-band Secret/resend-api (NOT committed).
|
||
# See documentation/runbooks/secrets.md. Overrides the empty ConfigMap placeholder.
|
||
- name: RESEND_API_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: resend-api
|
||
key: RESEND_API_KEY
|
||
# Operator/global bearer key — injected from the out-of-band Secret/report-api
|
||
# (NOT committed; documentation/runbooks/secrets.md). Deliberately NOT optional:
|
||
# a missing Secret must fail the pod Ready rather than boot an unauthenticatable
|
||
# hub with an empty bearer key. Create the Secret BEFORE syncing this manifest.
|
||
- name: REPORT_API_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: report-api
|
||
key: REPORT_API_KEY
|
||
# Decision 69 / R-821 (v0.127.0): the AES-256 key that seals every Storage Box sub-account
|
||
# password at rest. Out-of-band Secret/offsite-secret-key (NOT committed;
|
||
# documentation/runbooks/secrets.md). NOT optional: without it the hub refuses to store or
|
||
# use any sub-account password, so a missing Secret must fail the pod rather than boot a hub
|
||
# whose off-site registrar is silently off.
|
||
- name: OFFSITE_SECRET_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: offsite-secret-key
|
||
key: OFFSITE_SECRET_KEY
|
||
- name: REGISTRY_USERNAME
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: gitea-creds
|
||
key: username
|
||
- name: REGISTRY_TOKEN
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: gitea-creds
|
||
key: password
|
||
# S1 offsite connectivity: the WG peer-sync push channel (doc 06 §5 + runbook
|
||
# offsite-endpoint.md). Addr is dev-phase literal (the throwaway endpoint); the SSH
|
||
# private key + (non-secret) pinned host key come from Secret/wg-endpoint-ssh,
|
||
# created out-of-band in runbook step 6 — optional so the pod starts before it
|
||
# exists (the hub logs peer-sync disabled until then).
|
||
- name: WG_ENDPOINT_SSH_ADDR
|
||
value: "167.233.158.164:22"
|
||
- name: WG_ENDPOINT_SSH_USER
|
||
value: "felhom-peersync"
|
||
- name: WG_ENDPOINT_SSH_KEY_FILE
|
||
value: "/etc/hub-secrets/wg-endpoint-ssh/key"
|
||
- name: WG_ENDPOINT_SSH_HOSTKEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: wg-endpoint-ssh
|
||
key: hostkey
|
||
optional: true
|
||
# PBS DR tier (SLICE 1): the tenantsync channel — same endpoint + pinned host key as
|
||
# peersync (env above), its OWN private key from Secret/tenantsync (out-of-band,
|
||
# runbook offsite-endpoint.md §10). Optional: absent → the hub logs tenantsync disabled.
|
||
- name: TENANTSYNC_SSH_KEY_FILE
|
||
value: "/etc/hub-secrets/tenantsync/key"
|
||
# Agent-plane immediate-sync (Direction-2a, v0.59.0): the poke sender — same endpoint +
|
||
# pinned host key + peersync user as above, its OWN forced-command key from Secret/agent-poke
|
||
# (out-of-band; the ep0 authorized_keys line carries the PUBLIC half, command="felhom-poke").
|
||
# Optional: absent → the hub logs the poke disabled; agent-plane saves still reconcile in
|
||
# ≤15 min. See documentation/runbooks/offsite-endpoint.md (poke section).
|
||
- name: POKE_SSH_KEY_FILE
|
||
value: "/etc/hub-secrets/agent-poke/key"
|
||
# Offsite provisioning (SLICE 1+2): Hetzner Storage Box API token + the NUMERIC id of the
|
||
# pool box, from the out-of-band Secret/storagebox (NOT committed). The token MUST be scoped
|
||
# to the dedicated storage project — NEVER the shared-project token (it can touch ep0).
|
||
# HETZNER_POOL_BOX_ID is the numeric box id (console #id), not the box name. Optional so the
|
||
# pod starts before the secret exists (hub then logs offsite provisioning disabled).
|
||
- name: HETZNER_POOL_BOX_ID
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: storagebox
|
||
key: HETZNER_POOL_BOX_ID
|
||
optional: true
|
||
- name: HETZNER_TOKEN
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: storagebox
|
||
key: HETZNER_TOKEN
|
||
optional: true
|
||
# Non-secret; the hub defaults to fsn1 anyway — explicit for clarity.
|
||
- name: HETZNER_LOCATION
|
||
value: "fsn1"
|
||
resources:
|
||
requests:
|
||
memory: "64Mi"
|
||
cpu: "50m"
|
||
limits:
|
||
memory: "256Mi"
|
||
cpu: "500m"
|
||
volumeMounts:
|
||
- name: data
|
||
mountPath: /data
|
||
- name: config
|
||
mountPath: /etc/felhom-hub
|
||
- name: wg-endpoint-ssh
|
||
mountPath: /etc/hub-secrets/wg-endpoint-ssh
|
||
readOnly: true
|
||
- name: tenantsync
|
||
mountPath: /etc/hub-secrets/tenantsync
|
||
readOnly: true
|
||
- name: agent-poke
|
||
mountPath: /etc/hub-secrets/agent-poke
|
||
readOnly: true
|
||
# R-1326 (2026-10-27): the hub took 31–36 s to start („starting" → „Listening on :8080", 0.174.0 and 0.174.1),
|
||
# and the liveness probe alone killed it after ~65 s — a growing database would have turned a deploy into a
|
||
# restart loop. The startup probe allows up to 180 s (36 × 5 s, 5× the measured start) and holds the liveness
|
||
# probe off until its first success; after a normal start the liveness probe watches as before. A longer
|
||
# initialDelaySeconds would leave the hub unwatched for that long on EVERY start instead.
|
||
startupProbe:
|
||
httpGet:
|
||
path: /healthz
|
||
port: 8080
|
||
periodSeconds: 5
|
||
timeoutSeconds: 3
|
||
failureThreshold: 36
|
||
livenessProbe:
|
||
httpGet:
|
||
path: /healthz
|
||
port: 8080
|
||
initialDelaySeconds: 5
|
||
periodSeconds: 30
|
||
timeoutSeconds: 5
|
||
readinessProbe:
|
||
httpGet:
|
||
path: /healthz
|
||
port: 8080
|
||
initialDelaySeconds: 3
|
||
periodSeconds: 10
|
||
timeoutSeconds: 3
|
||
volumes:
|
||
- name: data
|
||
persistentVolumeClaim:
|
||
claimName: hub-data
|
||
- name: config
|
||
configMap:
|
||
name: hub-config
|
||
- name: wg-endpoint-ssh
|
||
secret:
|
||
secretName: wg-endpoint-ssh
|
||
optional: true
|
||
items:
|
||
- key: key
|
||
path: key
|
||
mode: 0400
|
||
- name: tenantsync
|
||
secret:
|
||
secretName: tenantsync
|
||
optional: true
|
||
items:
|
||
- key: key
|
||
path: key
|
||
mode: 0400
|
||
- name: agent-poke
|
||
secret:
|
||
secretName: agent-poke
|
||
optional: true
|
||
items:
|
||
- key: key
|
||
path: key
|
||
mode: 0400
|
||
|
||
# =============================================================================
|
||
# SERVICE
|
||
# =============================================================================
|
||
---
|
||
apiVersion: v1
|
||
kind: Service
|
||
metadata:
|
||
name: hub
|
||
namespace: felhom-system
|
||
labels:
|
||
app: hub
|
||
spec:
|
||
selector:
|
||
app: hub
|
||
ports:
|
||
- port: 8080
|
||
targetPort: 8080
|
||
name: http
|
||
|
||
# =============================================================================
|
||
# INGRESS — hub.felhom.eu
|
||
# =============================================================================
|
||
---
|
||
apiVersion: networking.k8s.io/v1
|
||
kind: Ingress
|
||
metadata:
|
||
name: hub
|
||
namespace: felhom-system
|
||
annotations:
|
||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||
nginx.ingress.kubernetes.io/proxy-body-size: "2m"
|
||
# Geo-restrict to Hungary (operator-only dashboard)
|
||
# NOTE: /api/v1/report must also be reachable — all customers are in HU
|
||
nginx.ingress.kubernetes.io/configuration-snippet: |
|
||
set $geo_allowed 0;
|
||
if ($remote_addr ~ "^192\.168\.") { set $geo_allowed 1; }
|
||
if ($remote_addr ~ "^10\.") { set $geo_allowed 1; }
|
||
if ($geoip2_country_code = "HU") { set $geo_allowed 1; }
|
||
if ($geoip2_country_code = "IE") { set $geo_allowed 1; }
|
||
if ($geo_allowed = 0) {
|
||
return 403 "Access restricted to Hungary";
|
||
}
|
||
spec:
|
||
ingressClassName: nginx-internal
|
||
tls:
|
||
- hosts:
|
||
- hub.felhom.eu
|
||
secretName: hub-felhom-eu-tls
|
||
rules:
|
||
- host: hub.felhom.eu
|
||
http:
|
||
paths:
|
||
- path: /
|
||
pathType: Prefix
|
||
backend:
|
||
service:
|
||
name: hub
|
||
port:
|
||
number: 8080 |