R-753: the box tells visitors apart — cloudflared at a fixed address, traefik trusts only it, the controller reads the hop traefik saw

- felhom-tunnel network 172.16.253.0/29 (ip-range .4/30): cloudflared alone at .2, traefik at .3; traefik's websecure
  trusts forwarded headers from 172.16.253.2/32 only, and every request passes felhom-forwarded@file, which removes
  the client-writable host/path/address headers (X-Forwarded-Host/-Uri/-Method/-Prefix, Forwarded, True-Client-Ip, …)
  and fixes X-Forwarded-Port to 443 (measured: Cloudflare passes a client's X-Forwarded-Host/-Port).
- EnsureBaseStack reconciles a RUNNING traefik/cloudflared whose rendered files changed (recreate), refuses a rewrite
  that would drop a certificate resolver, and moves cloudflared only once traefik is on the tunnel network.
- clientIP: believed only when the TCP peer is traefik; the rightmost X-Forwarded-For entry (the hop traefik saw);
  the tunnel hop → CF-Connecting-IP (the edge refuses a client-sent one, measured 403). rateKey: IPv6 per /64.
  Dashboard login, claim, share and escrow counters key on it; the setup gate logs it.
- Dashboard login messages: keys, informal voice, both languages.
Red-proofs: RP-A1 (leftmost hop), RP-A2 (shared tunnel key), RP-A3 (no reconcile) — felhom.eu audits/visitors-2026-10-01/A.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-01 21:01:42 +02:00
parent c1b123c649
commit 1e8d045815
19 changed files with 1006 additions and 83 deletions
+130 -20
View File
@@ -5,6 +5,7 @@ import (
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
"os"
"path/filepath"
"sort"
"strings"
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
@@ -39,7 +40,24 @@ func (m *Manager) EnsureBaseStack() error {
traefikDir := filepath.Join(base, "traefik")
var errs []string
if err := m.ensureTraefik(traefikDir); err != nil {
// R-753: the tunnel's own network, with cloudflared at a fixed address. Without it traefik and cloudflared keep the
// old shape (traefik trusts nothing) — the box still routes; it just cannot tell tunnel visitors apart.
tunnelOK := true
if err := m.ensureTunnelNetwork(); err != nil {
tunnelOK = false
errs = append(errs, fmt.Sprintf("tunnel network: %v", err))
}
// The forwarded-header clean-up BEFORE traefik.yml: the entrypoint names it, and a missing middleware would break
// every route on the box.
fwdOK := true
if err := m.ensureForwardedHeaders(traefikDir); err != nil {
fwdOK = false
errs = append(errs, fmt.Sprintf("forwarded headers: %v", err))
}
if !fwdOK {
m.logger.Printf("[WARN] [infra] traefik left as it is — its forwarded-header file could not be written")
} else if err := m.ensureTraefik(traefikDir, tunnelOK); err != nil {
errs = append(errs, fmt.Sprintf("traefik: %v", err))
}
@@ -60,7 +78,13 @@ func (m *Manager) EnsureBaseStack() error {
}
if m.cfg.Infrastructure.CFTunnelToken != "" {
if err := m.ensureCloudflared(filepath.Join(base, "cloudflared")); err != nil {
// cloudflared moves to the tunnel network only once traefik is on it (it reaches traefik by name there); while
// traefik is down a running cloudflared is left alone, so an outage never flips it back and forth.
traefikUp := containerRunning("traefik")
cfTunnel := tunnelOK && traefikUp && containerOnNetwork("traefik", infra.TunnelNetwork)
if !traefikUp && containerRunning("cloudflared") {
m.logger.Printf("[INFO] [infra] cloudflared left as it is while traefik is not running")
} else if err := m.ensureCloudflared(filepath.Join(base, "cloudflared"), cfTunnel); err != nil {
errs = append(errs, fmt.Sprintf("cloudflared: %v", err))
}
} else {
@@ -89,9 +113,36 @@ func (m *Manager) EnsureBaseStack() error {
return nil
}
func (m *Manager) ensureTraefik(dir string) error {
// ensureTraefik deploys traefik when it is not running, and RECONCILES a running one: when the rendered files differ
// from the ones on disk (a release changed the template — R-753 added the tunnel trust), it rewrites them and recreates
// the container, because traefik reads its static file only at start. Equal files → nothing (the healthy tick).
// A rewrite that would DROP the certificate resolver the running file has is refused (logged): the inputs that produce
// it (the customer's e-mail) are missing, and dropping it would cost the box its certificates.
// Pinned by TestEnsureTraefik_* (internal/stacks/infra_test.go).
func (m *Manager) ensureTraefik(dir string, tunnel bool) error {
files, err := infra.RenderTraefik(infra.TraefikData{
ACMEEmail: m.cfg.Customer.Email,
CFAPIToken: m.cfg.Infrastructure.CFAPIToken,
Tunnel: tunnel,
})
if err != nil {
return err
}
if containerRunning("traefik") {
return nil
changed := changedInfraFiles(dir, files)
if len(changed) == 0 {
return nil
}
if cur, err := os.ReadFile(filepath.Join(dir, "traefik.yml")); err == nil &&
strings.Contains(string(cur), "certResolver") && !strings.Contains(files["traefik.yml"].Content, "certResolver") {
m.logger.Printf("[WARN] [infra] traefik NOT reconciled: the new traefik.yml would drop the running certificate resolver (no customer e-mail in the config) — left as it is")
return nil
}
m.logger.Printf("[INFO] [infra] traefik config changed (%s, tunnel trust %v) — rewriting and recreating traefik (routing pauses a few seconds)", strings.Join(changed, ", "), tunnel)
if err := writeInfraFiles(dir, files); err != nil {
return err
}
return m.composeUp(dir, "--force-recreate")
}
m.logger.Printf("[INFO] [infra] deploying traefik → %s", dir)
if err := os.MkdirAll(filepath.Join(dir, "dynamic"), 0o755); err != nil {
@@ -110,32 +161,91 @@ func (m *Manager) ensureTraefik(dir string) error {
if err := os.Chmod(acme, 0o600); err != nil {
return fmt.Errorf("chmod acme.json: %w", err)
}
files, err := infra.RenderTraefik(infra.TraefikData{
ACMEEmail: m.cfg.Customer.Email,
CFAPIToken: m.cfg.Infrastructure.CFAPIToken,
})
if err := writeInfraFiles(dir, files); err != nil {
return err
}
return m.composeUp(dir)
}
// ensureCloudflared deploys cloudflared, or reconciles a running one whose compose changed (R-753 moved it to the tunnel
// network at a fixed address); compose recreates the container when its networks change. The tunnel reconnects in a
// few seconds. Pinned by TestEnsureCloudflared_*.
func (m *Manager) ensureCloudflared(dir string, tunnel bool) error {
files, err := infra.RenderCloudflared(infra.CloudflaredData{CFTunnelToken: m.cfg.Infrastructure.CFTunnelToken, Tunnel: tunnel})
if err != nil {
return err
}
if containerRunning("cloudflared") {
if len(changedInfraFiles(dir, files)) == 0 {
return nil
}
m.logger.Printf("[INFO] [infra] cloudflared compose changed (tunnel network %v) — recreating cloudflared (the tunnel reconnects in seconds)", tunnel)
} else {
m.logger.Printf("[INFO] [infra] deploying cloudflared → %s (tunnel network %v)", dir, tunnel)
}
if err := writeInfraFiles(dir, files); err != nil {
return err
}
return m.composeUp(dir)
}
func (m *Manager) ensureCloudflared(dir string) error {
if containerRunning("cloudflared") {
return nil
// ensureTunnelNetwork makes felhom-tunnel with its FIXED subnet (infra.TunnelSubnet). A network of that name with any
// other subnet is an error and is left alone: cloudflared could not take its address there, and traefik's trust would
// name an address nobody holds. Pinned by TestEnsureTunnelNetwork_*.
func (m *Manager) ensureTunnelNetwork() error {
inspect := func() (string, error) {
out, err := dockerexec.Command("docker", "network", "inspect", "--format",
"{{range .IPAM.Config}}{{.Subnet}} {{end}}", infra.TunnelNetwork).Output()
return strings.TrimSpace(string(out)), err
}
m.logger.Printf("[INFO] [infra] deploying cloudflared → %s", dir)
files, err := infra.RenderCloudflared(infra.CloudflaredData{CFTunnelToken: m.cfg.Infrastructure.CFTunnelToken})
if got, err := inspect(); err == nil {
if got == infra.TunnelSubnet {
return nil
}
return fmt.Errorf("docker network %s exists with subnet %q, want %s — left alone; the tunnel keeps its old shape", infra.TunnelNetwork, got, infra.TunnelSubnet)
}
m.logger.Printf("[INFO] [infra] creating docker network %s (%s)", infra.TunnelNetwork, infra.TunnelSubnet)
out, err := dockerexec.Command("docker", "network", "create", "--driver", "bridge",
"--subnet", infra.TunnelSubnet, "--ip-range", infra.TunnelIPRange, "--gateway", infra.TunnelGateway,
infra.TunnelNetwork).CombinedOutput()
if err != nil {
return err
if got, ierr := inspect(); ierr == nil && got == infra.TunnelSubnet {
return nil // created by a concurrent actor
}
return fmt.Errorf("network create %s: %s: %w", infra.TunnelNetwork, strings.TrimSpace(string(out)), err)
}
if err := writeInfraFiles(dir, files); err != nil {
return err
return nil
}
// ensureForwardedHeaders writes the forwarded-header clean-up middleware (infra.RenderForwardedHeaders) when its content
// changes. traefik.yml names it on the websecure entrypoint, so EnsureBaseStack writes it BEFORE traefik.yml.
func (m *Manager) ensureForwardedHeaders(traefikDir string) error {
dynDir := filepath.Join(traefikDir, "dynamic")
if err := os.MkdirAll(dynDir, 0o755); err != nil {
return fmt.Errorf("mkdir dynamic: %w", err)
}
return m.composeUp(dir)
path := filepath.Join(dynDir, "forwarded.yml")
want := infra.RenderForwardedHeaders()
if cur, err := os.ReadFile(path); err != nil || string(cur) != want {
if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
return fmt.Errorf("write forwarded headers: %w", err)
}
m.logger.Printf("[INFO] [infra] wrote the forwarded-header clean-up → %s (%s)", path, infra.ForwardedMiddleware)
}
return nil
}
// changedInfraFiles names the rendered files whose content differs from the file on disk (absent counts as different).
func changedInfraFiles(dir string, files map[string]infra.FileSpec) []string {
var changed []string
for name, spec := range files {
cur, err := os.ReadFile(filepath.Join(dir, name))
if err != nil || string(cur) != spec.Content {
changed = append(changed, name)
}
}
sort.Strings(changed)
return changed
}
func (m *Manager) ensureFileBrowser(dir string) error {
@@ -265,9 +375,9 @@ func (m *Manager) ensureTraefikNetwork() error {
return nil
}
// composeUp runs `docker compose up -d` in dir (DOMAIN injected by composeExecWithEnv).
func (m *Manager) composeUp(dir string) error {
out, err := m.composeExecWithEnv(dir, nil, "up", "-d")
// composeUp runs `docker compose up -d [extra…]` in dir (DOMAIN injected by composeExecWithEnv).
func (m *Manager) composeUp(dir string, extra ...string) error {
out, err := m.composeExecWithEnv(dir, nil, append([]string{"up", "-d"}, extra...)...)
if err != nil {
return fmt.Errorf("compose up: %s: %w", truncateStr(strings.TrimSpace(out), 300), err)
}
@@ -0,0 +1,257 @@
package stacks
import (
"io"
"log"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
)
// R-753 (`09` §3 decision 63, Part A): the base stack moves cloudflared onto its own network at a fixed address and
// makes traefik trust forwarded headers from that address only. These tests drive EnsureBaseStack's pieces against a
// STUB docker on PATH (never the real one — R-650) and a recorded compose seam, and assert the consequence on disk and
// in the commands run.
// stubDocker writes a fake `docker` into a temp dir on PATH. State lives in files under state/:
//
// running-<name> → `docker inspect --format {{.State.Running}} <name>` prints true
// nets-<name> → the networks `containerOnNetwork` sees (one per line)
// net-<network> → `docker network inspect --format … <network>` prints the file (else exit 1)
//
// `docker network create … --subnet S … <network>` writes net-<network> = S unless state/create-fails exists.
// Every call is appended to state/calls.
func stubDocker(t *testing.T) (state string) {
t.Helper()
if runtime.GOOS == "windows" {
t.Skip("the stub docker is a shell script")
}
dir := t.TempDir()
state = filepath.Join(dir, "state")
if err := os.MkdirAll(state, 0o755); err != nil {
t.Fatal(err)
}
script := `#!/bin/sh
S="` + state + `"
echo "$*" >> "$S/calls"
case "$1" in
inspect)
last=""; for a in "$@"; do last="$a"; done
case "$*" in
*State.Running*) if [ -f "$S/running-$last" ]; then echo true; else echo false; fi; exit 0;;
*NetworkSettings.Networks*) [ -f "$S/nets-$last" ] && cat "$S/nets-$last"; exit 0;;
esac;;
network)
last=""; for a in "$@"; do last="$a"; done
case "$2" in
inspect) if [ -f "$S/net-$last" ]; then cat "$S/net-$last"; exit 0; fi; echo "Error: no such network: $last" >&2; exit 1;;
create)
if [ -f "$S/create-fails" ]; then echo "Error response from daemon: Pool overlaps with other one on this address space" >&2; exit 1; fi
sub=""; prev=""; for a in "$@"; do [ "$prev" = "--subnet" ] && sub="$a"; prev="$a"; done
if [ -n "$sub" ]; then echo "$sub" > "$S/net-$last"; else echo "auto" > "$S/net-$last"; fi; exit 0;;
connect) exit 0;;
esac;;
esac
exit 0
`
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
return state
}
func touch(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
type composeCall struct {
dir string
args string
}
func tunnelTestManager(t *testing.T, email string) (*Manager, *[]composeCall) {
t.Helper()
cfg := &config.Config{}
cfg.Customer.Email = email
cfg.Infrastructure.CFTunnelToken = "tok-test"
m := &Manager{cfg: cfg, logger: log.New(io.Discard, "", 0)}
var calls []composeCall
m.composeExecFn = func(dir string, env map[string]string, args ...string) (string, error) {
calls = append(calls, composeCall{dir: dir, args: strings.Join(args, " ")})
return "", nil
}
return m, &calls
}
// The network is created with its FIXED subnet — the address traefik trusts must be one docker cannot hand elsewhere.
func TestEnsureTunnelNetwork_CreatesFixedSubnet(t *testing.T) {
state := stubDocker(t)
m, _ := tunnelTestManager(t, "")
if err := m.ensureTunnelNetwork(); err != nil {
t.Fatalf("ensureTunnelNetwork: %v", err)
}
got, _ := os.ReadFile(filepath.Join(state, "net-"+infra.TunnelNetwork))
if strings.TrimSpace(string(got)) != infra.TunnelSubnet {
t.Fatalf("network created with subnet %q, want %s", got, infra.TunnelSubnet)
}
calls, _ := os.ReadFile(filepath.Join(state, "calls"))
if !strings.Contains(string(calls), "--gateway "+infra.TunnelGateway) || !strings.Contains(string(calls), "--ip-range "+infra.TunnelIPRange) {
t.Fatalf("create did not fix the gateway: %s", calls)
}
}
// A network of that name with another subnet is NOT trusted and NOT touched: an error, and the caller keeps the old shape.
func TestEnsureTunnelNetwork_WrongSubnetIsAnError(t *testing.T) {
state := stubDocker(t)
touch(t, filepath.Join(state, "net-"+infra.TunnelNetwork), "172.30.0.0/16\n")
m, _ := tunnelTestManager(t, "")
err := m.ensureTunnelNetwork()
if err == nil || !strings.Contains(err.Error(), "172.30.0.0/16") {
t.Fatalf("want an error naming the wrong subnet, got %v", err)
}
calls, _ := os.ReadFile(filepath.Join(state, "calls"))
if strings.Contains(string(calls), "network create") || strings.Contains(string(calls), " rm ") {
t.Fatalf("a wrong-subnet network must be left alone; calls: %s", calls)
}
}
// THE CONSEQUENCE on a box that already runs traefik (every installed box): the new release rewrites traefik.yml with the
// tunnel trust and RECREATES traefik (static config is read only at start). A second tick with nothing changed does nothing.
func TestEnsureTraefik_ReconcilesARunningTraefik(t *testing.T) {
state := stubDocker(t)
touch(t, filepath.Join(state, "running-traefik"), "")
m, calls := tunnelTestManager(t, "owner@example.com")
dir := t.TempDir()
old, err := infra.RenderTraefik(infra.TraefikData{ACMEEmail: "owner@example.com"})
if err != nil {
t.Fatal(err)
}
// the pre-R-753 file: no trust, no middleware
oldYML := strings.Replace(old["traefik.yml"].Content, " http:\n middlewares:\n - "+infra.ForwardedMiddleware+"@file\n", " http:\n", 1)
touch(t, filepath.Join(dir, "traefik.yml"), oldYML)
touch(t, filepath.Join(dir, "docker-compose.yml"), old["docker-compose.yml"].Content)
if err := m.ensureTraefik(dir, true); err != nil {
t.Fatalf("ensureTraefik: %v", err)
}
yml, _ := os.ReadFile(filepath.Join(dir, "traefik.yml"))
if !strings.Contains(string(yml), `- "`+infra.TunnelAddr+`/32"`) {
t.Fatalf("traefik.yml was not rewritten with the tunnel trust:\n%s", yml)
}
cmp, _ := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
if !strings.Contains(string(cmp), "ipv4_address: "+infra.TunnelTraefikAddr) {
t.Fatalf("traefik's compose does not join %s:\n%s", infra.TunnelNetwork, cmp)
}
if len(*calls) != 1 || (*calls)[0].args != "up -d --force-recreate" {
t.Fatalf("want ONE `up -d --force-recreate`, got %+v", *calls)
}
if err := m.ensureTraefik(dir, true); err != nil {
t.Fatalf("second ensureTraefik: %v", err)
}
if len(*calls) != 1 {
t.Fatalf("an unchanged config must not recreate traefik again; calls %+v", *calls)
}
}
// A rewrite that would DROP the certificate resolver the running file has is refused (no e-mail in the config).
func TestEnsureTraefik_RefusesToDropTheCertResolver(t *testing.T) {
state := stubDocker(t)
touch(t, filepath.Join(state, "running-traefik"), "")
m, calls := tunnelTestManager(t, "") // no customer e-mail → the render has no resolver
dir := t.TempDir()
withACME, _ := infra.RenderTraefik(infra.TraefikData{ACMEEmail: "owner@example.com"})
touch(t, filepath.Join(dir, "traefik.yml"), withACME["traefik.yml"].Content)
if err := m.ensureTraefik(dir, true); err != nil {
t.Fatalf("ensureTraefik: %v", err)
}
yml, _ := os.ReadFile(filepath.Join(dir, "traefik.yml"))
if string(yml) != withACME["traefik.yml"].Content || len(*calls) != 0 {
t.Fatalf("the running file with a resolver must be left alone; calls %+v", *calls)
}
}
// cloudflared moves to the tunnel network at the fixed address — and is recreated by compose — only when asked.
func TestEnsureCloudflared_MovesToTheTunnelNetwork(t *testing.T) {
state := stubDocker(t)
touch(t, filepath.Join(state, "running-cloudflared"), "")
m, calls := tunnelTestManager(t, "")
dir := t.TempDir()
old, _ := infra.RenderCloudflared(infra.CloudflaredData{CFTunnelToken: "tok-test"})
touch(t, filepath.Join(dir, "docker-compose.yml"), old["docker-compose.yml"].Content)
if err := m.ensureCloudflared(dir, false); err != nil || len(*calls) != 0 {
t.Fatalf("unchanged old shape must do nothing; err %v calls %+v", err, *calls)
}
if err := m.ensureCloudflared(dir, true); err != nil {
t.Fatalf("ensureCloudflared: %v", err)
}
cmp, _ := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
if !strings.Contains(string(cmp), "ipv4_address: "+infra.TunnelAddr) || strings.Contains(string(cmp), "traefik-public") {
t.Fatalf("cloudflared must be ALONE on %s at %s:\n%s", infra.TunnelNetwork, infra.TunnelAddr, cmp)
}
if len(*calls) != 1 || (*calls)[0].args != "up -d" {
t.Fatalf("want one `up -d`, got %+v", *calls)
}
}
// The whole bring-up, in order: network → the header clean-up file → traefik (recreated with the trust) → cloudflared
// moved only because traefik is on the tunnel network. And when the network cannot be made, NOTHING moves and traefik
// keeps trusting nobody.
func TestEnsureBaseStack_TunnelOrder(t *testing.T) {
for _, tc := range []struct {
name string
createFail bool
}{{"network made", false}, {"network refused", true}} {
t.Run(tc.name, func(t *testing.T) {
state := stubDocker(t)
touch(t, filepath.Join(state, "net-traefik-public"), "172.18.0.0/16\n")
for _, c := range []string{"traefik", "cloudflared", "filebrowser", "felhom-controller"} {
touch(t, filepath.Join(state, "running-"+c), "")
}
if tc.createFail {
touch(t, filepath.Join(state, "create-fails"), "")
touch(t, filepath.Join(state, "nets-traefik"), "traefik-public\n")
} else {
touch(t, filepath.Join(state, "nets-traefik"), "traefik-public\n"+infra.TunnelNetwork+"\n")
}
touch(t, filepath.Join(state, "nets-felhom-controller"), "traefik-public\n")
m, calls := tunnelTestManager(t, "owner@example.com")
m.cfg.Paths.StacksDir = t.TempDir()
_ = m.EnsureBaseStack()
traefikDir := filepath.Join(m.cfg.Paths.StacksDir, "traefik")
if _, err := os.Stat(filepath.Join(traefikDir, "dynamic", "forwarded.yml")); err != nil {
t.Fatalf("the forwarded-header file must be written in either case: %v", err)
}
yml, _ := os.ReadFile(filepath.Join(traefikDir, "traefik.yml"))
cf, _ := os.ReadFile(filepath.Join(m.cfg.Paths.StacksDir, "cloudflared", "docker-compose.yml"))
trusts := strings.Contains(string(yml), "trustedIPs")
moved := strings.Contains(string(cf), "ipv4_address: "+infra.TunnelAddr)
if tc.createFail {
if trusts || moved {
t.Fatalf("no network → no trust and no move; trust %v moved %v", trusts, moved)
}
return
}
if !trusts || !moved {
t.Fatalf("network made → traefik trusts the tunnel and cloudflared moved; trust %v moved %v", trusts, moved)
}
var order []string
for _, c := range *calls {
order = append(order, filepath.Base(c.dir)+":"+c.args)
}
if len(order) < 2 || order[0] != "traefik:up -d --force-recreate" || order[1] != "cloudflared:up -d" {
t.Fatalf("traefik must be recreated BEFORE cloudflared moves; compose calls %v", order)
}
})
}
}