admin/felhom.eu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(hub,install): break-glass recovery vault + mgmt_plane surfacing (TASK G1)

Browse Source 
Hub half of the management-plane break-glass (prereq for felhom-sshd/H1; agent
half = felhom-agent v0.71.0). Closes SPIKE-felhom-sshd §8/#9.

- store.host_recovery + methods: per-host root@pam console password, at-rest,
  operator-retrievable (the PVE-web-console fallback when sshd + auto-heal both fail).
- API: PUT /hosts/{id}/recovery-credential (self-scoped, day-0 vaults) + GET
  /admin/hosts/{id}/recovery-credential (global key only). Secret never logged
  (red-proofed).
- monitor/host_mgmtplane: parses the agent mgmt_plane stanza, raises
  mgmt_plane_healed WARNING on a new privsep_healed_at (recurring clobber surfaces
  before lockout; complements host_staleness).
- host-install: step_break_glass generates a strong root@pam password (openssl
  rand, never logged/filed — stdin to chpasswd + curl), vaults via host key;
  idempotent unless --rotate-recovery. Installs the G1 host artifacts (tmpfiles +
  agent-independent watchdog timer), RuntimeDirectory-guarded; uninstall removes them.

Hub v0.34.0. Non-hollow tests + red-proofs; full suite green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
authentik Default Admin
2026-07-05 19:03:18 +02:00
parent 2f97ce31dd
commit 05d81810d4
10 changed files with 739 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/felhom-host-install.sh
+122
View File
@@ -153,6 +153,7 @@ UNINSTALL=false # --uninstall: local host teardown (destroy guest + rem
REMOVE_GOLDEN=false # --remove-golden: also delete the golden vzdump during --uninstall
ADOPT_POOL=false # --adopt-pool: retrofit an EXISTING Felhom guest into the felhom pool (non-destructive)
RESCOPE_ACL=false # --rescope-acl: migrate an existing install from the broad-/ token to the scoped ACL
ROTATE_RECOVERY=false # --rotate-recovery: regenerate + re-vault the break-glass root@pam password (TASK G1)
# --- Gitea (artifact source) + agent install model (BUNDLE slice) ---
GITEA_BASE="https://gitea.dooplex.hu"
@@ -582,6 +583,23 @@ run_uninstall() {
if [[ -f "$agent_cfg" ]]; then run rm -f "$agent_cfg"; else log_skip " $agent_cfg already absent"; fi
run rmdir "$(dirname "$agent_cfg")" 2>/dev/null || true
# 4b2. Management-plane break-glass (TASK G1): timer+oneshot+script+tmpfiles. Stop/disable the
# timer, remove all four artifacts + the runtime heal-marker. We do NOT `rmdir /run/sshd` —
# the stock sshd needs it; leaving the (now unit-less) dir in place is correct (a bare kernel
# /run tmpfs recreates it empty on next boot anyway). Tolerate-absent throughout.
if systemctl list-unit-files felhom-mgmt-watchdog.timer >/dev/null 2>&1; then
systemctl is-active --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl stop felhom-mgmt-watchdog.timer
systemctl is-enabled --quiet felhom-mgmt-watchdog.timer 2>/dev/null && run systemctl disable felhom-mgmt-watchdog.timer
else
log_skip " felhom-mgmt-watchdog.timer not loaded — skip stop/disable"
fi
run systemctl reset-failed felhom-mgmt-watchdog.service 2>/dev/null || true
local wda
for wda in /etc/systemd/system/felhom-mgmt-watchdog.service /etc/systemd/system/felhom-mgmt-watchdog.timer \
/usr/local/sbin/felhom-mgmt-watchdog /etc/tmpfiles.d/felhom-privsep.conf /run/felhom-mgmt-watchdog.healed; do
if [[ -e "$wda" ]]; then run rm -f "$wda"; fi
done
# 4c. Shared-parent unit + wrapper + /mnt/felhom-drives (agent-installed at runtime; drill R2).
# Stop/disable, remove unit + script, unbind + remove the (empty) parent dir. Tolerate-absent.
if systemctl list-unit-files felhom-shared-parent.service 2>/dev/null | grep -q felhom-shared-parent; then
@@ -782,6 +800,7 @@ while [[ $# -gt 0 ]]; do
--remove-golden) REMOVE_GOLDEN=true; shift ;;
--adopt-pool) ADOPT_POOL=true; shift ;;
--rescope-acl) RESCOPE_ACL=true; shift ;;
--rotate-recovery) ROTATE_RECOVERY=true; shift ;;
--acl-storages) read -ra PVE_STORAGES <<< "$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--resume) RESUME=true; shift ;;
@@ -1141,6 +1160,54 @@ step_enroll() {
_state_mark enroll
}
#-------------------------------------------------------------------------------
# STEP 4b — break-glass credential (TASK G1): generate + set + vault the root@pam console password
#-------------------------------------------------------------------------------
# The human fallback for when BOTH the sshd path AND the agent-independent auto-heal (layers 1+2) have
# failed: a strong root@pam password lets the operator reach the PVE WEB CONSOLE (pveproxy :8006 — a
# failure domain distinct from sshd) and run the one-line /run/sshd fix. Generated with strong entropy,
# set via chpasswd, and vaulted to the hub over the enroll-authenticated channel (host api_key). The
# password is NEVER logged, printed, or written to any file — it goes stdin→chpasswd and stdin→curl
# only (SPIKE-felhom-sshd finding #9 / TASK G1 trap 3). Idempotent: skipped if already vaulted unless
# --rotate-recovery (a re-set would strand the operator's saved copy).
step_break_glass() {
log_step "4b/8 break-glass credential (root@pam console password → hub vault)"
if [[ -z "${HOST_ID:-}" || -z "${HOST_API_KEY:-}" ]]; then
log_warn " no host_id/api_key (enroll skipped?) — cannot vault a recovery credential; skipping"
return 0
fi
if $DRY_RUN; then
log_dry "openssl rand → strong root@pam password (never logged) ; chpasswd ; PUT $HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential (Bearer host key)"
_state_mark break_glass; return 0
fi
if _state_has break_glass && ! $ROTATE_RECOVERY; then
log_skip " recovery credential already vaulted (use --rotate-recovery to regenerate)"
return 0
fi
# Strong password: 24 url-safe bytes (~144 bits). Kept ONLY in a local shell var, never on disk.
local newpw
newpw=$(openssl rand -base64 24 2>/dev/null | tr -d '\n' | tr '+/' '-_')
[[ ${#newpw} -ge 24 ]] || die "failed to generate a strong recovery password"
# Set root@pam (= the Linux root user on PVE) via chpasswd on STDIN — no argv, no log.
if ! printf 'root:%s\n' "$newpw" | chpasswd 2>/dev/null; then
newpw="" # scrub
die "chpasswd failed to set the root@pam recovery password"
fi
# Vault to the hub over the host-key-authenticated channel; password only on stdin (-d @-).
local code
code=$(printf '{"username":"root@pam","password":"%s"}' "$newpw" \
| curl -sS -o /dev/null -w '%{http_code}' -X PUT \
"$HUB_URL/api/v1/hosts/$HOST_ID/recovery-credential" \
-H "Authorization: Bearer $HOST_API_KEY" -H 'Content-Type: application/json' -d @- 2>/dev/null)
newpw="" # scrub the plaintext from the shell var the moment it is vaulted
case "$code" in
200) log_success " root@pam password set + vaulted to the hub (retrieve via the operator /admin path; never logged here)" ;;
401|403) die "recovery-credential vault rejected ($code) — host key/authorization problem" ;;
*) die "recovery-credential vault failed (HTTP $code)" ;;
esac
_state_mark break_glass
}
#-------------------------------------------------------------------------------
# STEP 5 — agent install: fetch+verify the binary, ensure the service user, sudoers, unit
#-------------------------------------------------------------------------------
@@ -1331,9 +1398,63 @@ step_agent_install() {
fi
rm -f "$rbtmp"
fi
# Management-plane break-glass layers 1+2 (TASK G1). Three artifacts that keep the host reachable
# even if a second sshd (H1) removes the SHARED /run/sshd privsep dir (SPIKE-felhom-sshd §8):
# • felhom-privsep.conf (tmpfiles) — layer 1: /run/sshd is boot-persistent, owned by no unit.
# • felhom-mgmt-watchdog (script) — layer 2 heal action (recreate dir + reset-failed sshd).
# • .service + .timer — run it every ~60s, AGENT-INDEPENDENTLY (heals with the
# agent down — the whole point; trap 1).
# Non-fatal if the agent repo predates them (raw fetch 404s → break-glass just stays manual).
# HARD GUARD: refuse ANY fetched unit that declares RuntimeDirectory= — that directive is the very
# incident G1 closes (a second sshd's `RuntimeDirectory=sshd` removed the shared /run/sshd).
install_mgmt_watchdog
_state_mark agent_install
}
# install_mgmt_watchdog fetches + installs the G1 break-glass host artifacts (idempotent; enables the
# timer). Split out for readability; called from step_agent_install. Every unit is RuntimeDirectory-
# guarded (trap 2). Non-fatal on a repo that predates the artifacts.
install_mgmt_watchdog() {
if $DRY_RUN; then
log_dry "fetch configs/felhom-privsep.tmpfiles -> /etc/tmpfiles.d/felhom-privsep.conf ; systemd-tmpfiles --create"
log_dry "fetch configs/felhom-mgmt-watchdog.sh -> /usr/local/sbin/felhom-mgmt-watchdog (0755)"
log_dry "fetch configs/felhom-mgmt-watchdog.{service,timer} -> /etc/systemd/system/ ; enable --now felhom-mgmt-watchdog.timer"
return 0
fi
local wdtmp; wdtmp=$(mktemp -t felhom-mgmt-wd.XXXXXX)
if ! fetch_raw "configs/felhom-mgmt-watchdog.sh" "$wdtmp" 2>/dev/null; then
log_skip " mgmt-watchdog artifacts not in the agent repo yet — break-glass auto-heal stays manual"
rm -f "$wdtmp"; return 0
fi
sh -n "$wdtmp" || { rm -f "$wdtmp"; die "fetched felhom-mgmt-watchdog.sh failed sh -n — refusing to install"; }
install -m 0755 -o root -g root "$wdtmp" /usr/local/sbin/felhom-mgmt-watchdog
rm -f "$wdtmp"
# tmpfiles (layer 1) — install + create now so /run/sshd is guaranteed present immediately.
local tftmp; tftmp=$(mktemp -t felhom-privsep.XXXXXX)
fetch_raw "configs/felhom-privsep.tmpfiles" "$tftmp"
install -m 0644 -o root -g root "$tftmp" /etc/tmpfiles.d/felhom-privsep.conf
rm -f "$tftmp"
systemd-tmpfiles --create /etc/tmpfiles.d/felhom-privsep.conf 2>/dev/null || true
# units (layer 2) — RuntimeDirectory guard on BOTH before install (trap 2 / the incident cause).
local svctmp tmrtmp; svctmp=$(mktemp -t felhom-wd-svc.XXXXXX); tmrtmp=$(mktemp -t felhom-wd-tmr.XXXXXX)
fetch_raw "configs/felhom-mgmt-watchdog.service" "$svctmp"
fetch_raw "configs/felhom-mgmt-watchdog.timer" "$tmrtmp"
if grep -qiE '^[[:space:]]*RuntimeDirectory[[:space:]]*=' "$svctmp" "$tmrtmp"; then
rm -f "$svctmp" "$tmrtmp"
die "mgmt-watchdog unit declares RuntimeDirectory= — that is the incident G1 fixes; refusing to install"
fi
install -m 0644 -o root -g root "$svctmp" /etc/systemd/system/felhom-mgmt-watchdog.service
install -m 0644 -o root -g root "$tmrtmp" /etc/systemd/system/felhom-mgmt-watchdog.timer
rm -f "$svctmp" "$tmrtmp"
systemctl daemon-reload
systemctl enable --now felhom-mgmt-watchdog.timer >/dev/null 2>&1 || true
log_success " installed break-glass layers 1+2 (tmpfiles /run/sshd + agent-independent watchdog timer)"
}
#-------------------------------------------------------------------------------
# STEP 6 — write agent config + ensure service healthy
#-------------------------------------------------------------------------------
@@ -1611,6 +1732,7 @@ fi
should_skip token || step_token
should_skip grows || step_grows
should_skip enroll || step_enroll
should_skip break_glass || step_break_glass
should_skip agent_install || step_agent_install
should_skip agent_config || step_agent_config
should_skip golden || step_golden