felhom.eu/scripts/iso/felhom-bootstrap.sh

#!/bin/bash
#===============================================================================
# felhom-bootstrap.sh — invoked by felhom-bootstrap.service, retried until host-install succeeds.
#
# ONE unit, TWO modes, decided by the env:
#   DIRECT  (env has FELHOM_CUSTOMER_ID + FELHOM_RETRIEVAL_PASSPHRASE) — the slice-A path, unchanged:
#           fetch felhom-host-install.sh from the PUBLIC channel -> run it unattended with the
#           customer's retrieval passphrase -> on rc 0 write the done-flag + disable + shred the env.
#   PAIRING (R-21 slice C — the GENERIC secret-free ISO, no customer-id/passphrase in the env):
#           register this box as an UNCLAIMED appliance at the hub (uuid + MAC set + SSH host keys +
#           hw), receive a one-per-registration APPLIANCE TOKEN (0600), then POLL for the operator's
#           bind. ONE delivery hands over customer-id + retrieval passphrase; the bootstrap WRITES
#           them into the env (0600) and FALLS THROUGH to the DIRECT path — so every later retry is a
#           plain direct install (the delivery is one-shot; the box must not depend on re-fetching it).
#
# The PAIRING wait polls INSIDE this script (v1.21.0, R-33). It used to be systemd's
# Restart=on-failure/RestartSec=30 — one poll per invocation, exiting non-zero until the bind landed.
# That worked, but every 30s systemd printed `Failed to start Felhom host bootstrap` on the physical
# console the CUSTOMER is watching: 52 FAILED lines in ~11 minutes during the 2026-07-18 rehearsal,
# while nothing was wrong (the box was correctly waiting to be bound). Waiting is not failing, so it
# must not be reported as failure. The unit now stays in `activating` and the loop sleeps between
# polls; the unit's Restart= machinery is kept for the DIRECT path and for genuine crashes.
# REQUIRES TimeoutStartSec=infinity in the unit — a Type=oneshot ExecStart is otherwise killed at
# DefaultTimeoutStartSec (90s), which would silently reintroduce the restart spam.
#
# Retry-vs-resume (source-verified, encoded ONCE): felhom-host-install.sh v1.11.3 makes --resume safe
# — its producer steps re-run every pass. FIRST direct attempt is plain; any later attempt that finds
# the install state file adds --resume. State file: /var/lib/felhom-install/state.json.
#
# NOT production-generic: this is the R-21 bare-metal first-boot bootstrap. It does NOT modify
# felhom-host-install.sh; it only invokes it.
#===============================================================================
# Deliberately NOT `set -e`: we must capture exit codes and exit on our own terms.
set -uo pipefail

ENV_FILE=/etc/felhom/bootstrap.env
DONE_FLAG=/etc/felhom/.bootstrap-done
STATE_FILE=/var/lib/felhom-install/state.json
PASS_FILE=/run/felhom-bootstrap-pass
SCRIPT_TMP=/run/felhom-host-install.sh
TOKEN_FILE=/etc/felhom/appliance-token   # PAIRING: the box's only pre-day-0 credential (0600, persists reboots)
PAIRING_CODE_FILE=/etc/felhom/appliance-pairing-code  # R-27: non-secret pairing code shown on the console

# PAIRING wait cadence (v1.21.0, R-33). POLL_INTERVAL keeps the hub-side rate identical to the old
# RestartSec=30, so nothing downstream changes; the other two only govern how often we SPEAK.
POLL_INTERVAL=30      # seconds between polls
BANNER_EVERY=10       # re-print the console banner every N cycles (10 x 30s = 5 min)
HEARTBEAT_EVERY=20    # journal heartbeat every N cycles (20 x 30s = 10 min)

log() { echo "felhom-bootstrap: $*"; }

# print_pairing_banner (R-27, v0.66.0) — show the pairing code prominently on the physical console while
# the box waits to be bound, so the customer can read it into the self-bind page. Non-secret (the bind
# still requires the customer's retrieval passphrase). A hub older than v0.66.0 sends no code → no banner
# (the box stays operator-bind-only — graceful, no behavior change).
print_pairing_banner() {
    local code; code=$(cat "$PAIRING_CODE_FILE" 2>/dev/null)
    [[ -n "$code" ]] || return 0
    { printf '\n================================================\n'
      printf '  Felhom — a doboz készen áll, és a párosításra vár.\n\n'
      printf '  Párosító kód:  %s\n\n' "$code"
      printf '  Nyisd meg az e-mailben kapott linket, és add meg\n'
      printf '  ezt a kódot és a jelszavadat.\n\n'
      printf '  Ez a képernyő magától frissül — nincs teendő a\n'
      printf '  doboznál, és nyugodtan itt hagyhatod bekapcsolva.\n'
      printf '================================================\n\n'
    } > /dev/console 2>/dev/null || printf 'Párosító kód: %s\n' "$code"
}

cleanup_pass() { [[ -e "$PASS_FILE" ]] && { shred -u "$PASS_FILE" 2>/dev/null || rm -f "$PASS_FILE"; }; return 0; }
trap cleanup_pass EXIT

# Belt-and-suspenders: the unit already has ConditionPathExists=!done, but guard here too.
if [[ -e "$DONE_FLAG" ]]; then
    log "done-flag present ($DONE_FLAG) — nothing to do"
    exit 0
fi

# --- env (may be absent in the generic ISO; a non-secret pairing env can still set FELHOM_HUB_URL) --
FELHOM_CUSTOMER_ID=""; FELHOM_MODE=""; FELHOM_RETRIEVAL_PASSPHRASE=""
FELHOM_HUB_URL=""; FELHOM_INSTALL_URL=""; FELHOM_EXTRA_ARGS=""
if [[ -r "$ENV_FILE" ]]; then
    # shellcheck disable=SC1090
    source "$ENV_FILE"
fi
HUB_URL="${FELHOM_HUB_URL:-https://hub.felhom.eu}"
INSTALL_URL="${FELHOM_INSTALL_URL:-https://felhom.eu/scripts/felhom-host-install.sh}"

# =====================================================================================================
# DIRECT mode — fetch + run host-install with the customer passphrase (slice A, unchanged behaviour).
# =====================================================================================================
run_direct() {
    for var in FELHOM_CUSTOMER_ID FELHOM_MODE FELHOM_RETRIEVAL_PASSPHRASE; do
        if [[ -z "${!var:-}" ]]; then
            log "ERROR: $var is unset/empty (direct mode) — refusing to guess"
            exit 1
        fi
    done

    log "fetching host-install: $INSTALL_URL"
    if ! curl -fsSL --max-time 60 "$INSTALL_URL" -o "$SCRIPT_TMP"; then
        log "ERROR: host-install fetch failed (no network yet?) — unit will retry"
        exit 1
    fi
    if [[ ! -s "$SCRIPT_TMP" ]]; then
        log "ERROR: fetched host-install is empty — unit will retry"
        exit 1
    fi

    ( umask 077; printf '%s' "$FELHOM_RETRIEVAL_PASSPHRASE" > "$PASS_FILE" )

    local args=(--customer-id "$FELHOM_CUSTOMER_ID" --mode "$FELHOM_MODE" --hub-url "$HUB_URL" --passphrase-file "$PASS_FILE")
    if [[ -f "$STATE_FILE" ]]; then
        log "prior install state present ($STATE_FILE) -> adding --resume (host-install v1.11.3: producers re-run, safe)"
        args+=(--resume)
    fi
    local extra
    read -ra extra <<< "${FELHOM_EXTRA_ARGS:-}"

    log "running host-install (customer=${FELHOM_CUSTOMER_ID} mode=${FELHOM_MODE} hub=${HUB_URL})"
    bash "$SCRIPT_TMP" "${args[@]}" "${extra[@]}"
    local rc=$?
    cleanup_pass

    if [[ $rc -eq 0 ]]; then
        log "host-install SUCCESS — writing done-flag, disabling unit, scrubbing secrets"
        install -d -m 0755 "$(dirname "$DONE_FLAG")"
        : > "$DONE_FLAG"; chmod 0644 "$DONE_FLAG"
        systemctl disable felhom-bootstrap.service 2>/dev/null || true
        # Reduce secret-at-rest: the box is enrolled; the passphrase (and the appliance token) are done.
        shred -u "$ENV_FILE" 2>/dev/null || rm -f "$ENV_FILE"
        [[ -e "$TOKEN_FILE" ]] && { shred -u "$TOKEN_FILE" 2>/dev/null || rm -f "$TOKEN_FILE"; }
        exit 0
    fi

    log "host-install FAILED rc=${rc} — unit will retry in 30s"
    exit "$rc"
}

# =====================================================================================================
# PAIRING mode — register the unclaimed appliance, then ONE poll per invocation until the bind delivers.
# =====================================================================================================

# gather_identity_json builds the registration payload. Keying is (SMBIOS UUID, MAC set) — the N100 DMI
# verdict is that serials are unusable ("Default string"), so only the uuid + physical MAC set are
# trusted; hw is a non-keyed summary. python3 ships with PVE and JSON-encodes robustly.
gather_identity_json() {
    local uuid; uuid=$(tr -d '\n' < /sys/class/dmi/id/product_uuid 2>/dev/null)
    local product; product=$(tr -d '\n' < /sys/class/dmi/id/product_name 2>/dev/null)
    local mem_kb; mem_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo 2>/dev/null)
    local cpu; cpu=$(awk -F: '/model name/{print $2; exit}' /proc/cpuinfo 2>/dev/null | sed 's/^ *//')

    local macs=()
    local d n m
    for d in /sys/class/net/*; do
        n=$(basename "$d")
        [[ "$n" == "lo" ]] && continue
        [[ -e "$d/device" ]] || continue   # physical NICs only (skip bridges/veth/wg)
        m=$(cat "$d/address" 2>/dev/null)
        [[ -n "$m" && "$m" != "00:00:00:00:00:00" ]] && macs+=("$m")
    done

    local keys=()
    local f
    for f in /etc/ssh/ssh_host_*_key.pub; do
        [[ -f "$f" ]] && keys+=("$(cat "$f")")
    done

    UUID_G="$uuid" PRODUCT_G="$product" CPU_G="$cpu" MEM_G="$mem_kb" \
    MACS_G="$(printf '%s\n' "${macs[@]}")" KEYS_G="$(printf '%s\n' "${keys[@]}")" \
    python3 - <<'PY'
import json, os
def lines(v): return [x for x in (v or "").splitlines() if x.strip()]
print(json.dumps({
    "uuid": os.environ.get("UUID_G",""),
    "macs": lines(os.environ.get("MACS_G","")),
    "ssh_host_pubkeys": lines(os.environ.get("KEYS_G","")),
    "hw": {"product": os.environ.get("PRODUCT_G",""),
           "cpu": os.environ.get("CPU_G",""),
           "mem_kb": int(os.environ.get("MEM_G") or 0)},
}))
PY
}

register_appliance() {
    # Register once and persist the token. Returns 0 on success (or if already registered), 1 on a
    # transient failure the caller should simply retry — no network yet is the common case on a box
    # that has only just booted, and it is not an error worth telling the customer about.
    [[ -s "$TOKEN_FILE" ]] && return 0

    local payload; payload=$(gather_identity_json)
    if [[ -z "$payload" || "$payload" != *'"uuid"'* ]]; then
        log "could not gather appliance identity yet — retrying"
        return 1
    fi
    log "registering unclaimed appliance at the hub"
    local resp; resp=$(curl -fsS --max-time 30 -X POST \
        -H 'Content-Type: application/json' --data "$payload" \
        "$HUB_URL/api/v1/appliance/register" 2>/dev/null)
    if [[ $? -ne 0 || -z "$resp" ]]; then
        log "registration did not go through (no network yet?) — retrying"
        return 1
    fi
    local token; token=$(printf '%s' "$resp" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("appliance_token",""))' 2>/dev/null)
    if [[ -z "$token" ]]; then
        log "registration returned no appliance token — retrying"
        return 1
    fi
    ( umask 077; printf '%s' "$token" > "$TOKEN_FILE" )
    # R-27 (v0.66.0): persist the non-secret pairing code (absent on a pre-v0.66.0 hub — tolerated).
    local pcode; pcode=$(printf '%s' "$resp" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("pairing_code",""))' 2>/dev/null)
    if [[ -n "$pcode" ]]; then
        printf '%s' "$pcode" > "$PAIRING_CODE_FILE"
    fi
    log "registered — appliance token stored (0600); waiting for the operator or a customer self-bind"
    return 0
}

run_pairing() {
    log "PAIRING mode (generic ISO, no baked customer/passphrase) — hub=$HUB_URL"

    # The wait lives HERE, not in systemd's restart loop (v1.21.0, R-33). Waiting to be bound is the
    # NORMAL state of a freshly installed box and must look like it — on the console and in the
    # journal alike. Only a genuine crash should ever surface as a unit failure.
    local cycle=0
    while true; do
        if ! register_appliance; then
            sleep "$POLL_INTERVAL"; continue
        fi

        # Re-show the code periodically, not every cycle: the customer may walk up at any time, but a
        # banner every 30s is its own kind of noise.
        if (( cycle % BANNER_EVERY == 0 )); then
            print_pairing_banner
        fi

        local token; token=$(cat "$TOKEN_FILE")
        local body code
        body=$(curl -sS --max-time 30 -o - -w '\n%{http_code}' \
            -H "Authorization: Bearer $token" "$HUB_URL/api/v1/appliance/poll" 2>/dev/null)
        code="${body##*$'\n'}"
        body="${body%$'\n'*}"

        case "$code" in
        200)
            log "bind DELIVERED — writing credentials to the env and switching to direct install"
            # Parse the one-shot delivery into shell-safe env assignments (never echo the passphrase).
            local envtext
            envtext=$(printf '%s' "$body" | python3 -c '
import json, sys, shlex
d = json.load(sys.stdin)
def emit(k, v): print("%s=%s" % (k, shlex.quote(v or "")))
emit("FELHOM_CUSTOMER_ID", d.get("customer_id"))
emit("FELHOM_RETRIEVAL_PASSPHRASE", d.get("retrieval_passphrase"))
emit("FELHOM_MODE", d.get("mode") or "appliance")
emit("FELHOM_EXTRA_ARGS", d.get("extra_args"))
')
            if [[ -z "$envtext" || "$envtext" != *FELHOM_RETRIEVAL_PASSPHRASE=* ]]; then
                log "ERROR: delivery parse failed — unit will retry"
                exit 1
            fi
            # Persist as the direct-mode env (0600) so EVERY later retry is a plain direct install
            # (the delivery was one-shot; a second poll returns 410).
            install -d -m 0755 "$(dirname "$ENV_FILE")"
            ( umask 077
              { printf '%s\n' "$envtext"
                printf 'FELHOM_HUB_URL=%q\n' "$HUB_URL"
                printf 'FELHOM_INSTALL_URL=%q\n' "$INSTALL_URL"
              } > "$ENV_FILE" )
            chmod 0600 "$ENV_FILE"
            # Re-source + fall through to the direct install in THIS same invocation.
            # shellcheck disable=SC1090
            source "$ENV_FILE"
            run_direct
            ;;  # run_direct exits
        204)
            # The expected state for as long as nobody has bound the box. Log it once, then only on
            # a slow heartbeat — an operator reading the journal still sees liveness without the
            # every-30s wall of text that made the real signal hard to find.
            if (( cycle == 0 )); then
                log "not bound yet — polling every ${POLL_INTERVAL}s until the operator or a customer self-bind lands (this is the normal waiting state, not an error)"
            elif (( cycle % HEARTBEAT_EVERY == 0 )); then
                log "still waiting to be bound ($(( cycle * POLL_INTERVAL / 60 ))m elapsed; polling continues)"
            fi
            ;;
        410)
            # The delivery was consumed but we hold no env — a rare crash window. Exiting hands the
            # box back to systemd, whose restart re-runs us from a clean slate.
            log "ERROR: delivery already consumed but no local env — exiting so the unit restarts (rare crash-window)"
            exit 1
            ;;
        404)
            if (( cycle % HEARTBEAT_EVERY == 0 )); then
                log "appliance token not recognized (discarded, or the hub has no record) — still retrying"
            fi
            ;;
        *)
            if (( cycle % HEARTBEAT_EVERY == 0 )); then
                log "poll returned HTTP ${code:-none} — still retrying"
            fi
            ;;
        esac

        sleep "$POLL_INTERVAL"
        cycle=$(( cycle + 1 ))
    done
}

# --- mode selection -------------------------------------------------------------------------------
if [[ -n "$FELHOM_CUSTOMER_ID" && -n "$FELHOM_RETRIEVAL_PASSPHRASE" ]]; then
    run_direct
else
    run_pairing
fi