#!/bin/sh # felhom-mgmt-watchdog — the DUMB, agent-INDEPENDENT management-plane healer (TASK G1). # # WHY THIS EXISTS: /run/sshd is OpenSSH's compiled-in privilege-separation directory, SHARED by # every sshd on the host. If it goes missing, sshd's listener stays up but every new session RESETS # right after SSH2_MSG_KEXINIT — a management-plane lockout on an otherwise-healthy box # (SPIKE-felhom-sshd-2026-07-05 §8, caused live by a second unit's `RuntimeDirectory=sshd` cleanup). # The tmpfiles.d entry (felhom-privsep.tmpfiles) PREVENTS the known cause; this script AUTO-HEALS any # novel cause with ZERO login and ZERO dependency on the felhom-agent binary (so it self-corrects even # when the agent is down — the whole point). It is run by felhom-mgmt-watchdog.timer every ~60s. # # CONTRACT (all four are load-bearing): # 1. STAT-FIRST / IDEMPOTENT — a HEALTHY /run/sshd is NEVER touched (no mkdir/chmod/chown call), so # steady state is zero mutation and the dir's mtime is preserved. Only a MISSING or # wrong-mode/owner dir is corrected. # 2. reset-failed the STOCK sshd ONLY when it is in the `failed` state (a start-limit lockout) — # never otherwise, and NEVER `restart` it (mkdir alone restores it; PID stays, spike-proven). # 3. HEAL MARKER — writes an RFC3339 UTC timestamp to $MARKER only on a real heal, so the agent can # REPORT the condition to the hub (a recurring clobber must surface BEFORE it becomes a lockout). # 4. NO `RuntimeDirectory=` anywhere in this feature (that IS the incident cause) — the unit that # runs this script must not declare one either. # # Touches host /run + the stock sshd unit ONLY — no guests (pool-scoping is moot here). Fail-safe: # any single step failing is logged and does not abort the others (`|| true`); the next tick retries. set -u PRIVSEP=/run/sshd MARKER=/run/felhom-mgmt-watchdog.healed SSHD_UNIT=ssh.service # Debian/PVE stock sshd unit name (sshd.service is an alias) healed=0 log() { logger -t felhom-mgmt-watchdog "$*" 2>/dev/null || true; } # 1. Privsep dir — stat-first: create only when missing; correct mode/owner only when wrong. if [ ! -d "$PRIVSEP" ]; then if mkdir -p "$PRIVSEP" && chown root:root "$PRIVSEP" && chmod 0755 "$PRIVSEP"; then healed=1 log "recreated missing privsep dir $PRIVSEP (0755 root:root)" else log "ERROR: failed to recreate $PRIVSEP" fi else mode=$(stat -c %a "$PRIVSEP" 2>/dev/null || echo "") owner=$(stat -c %U:%G "$PRIVSEP" 2>/dev/null || echo "") if [ "$mode" != "755" ]; then chmod 0755 "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP mode ($mode -> 755)" fi if [ "$owner" != "root:root" ]; then chown root:root "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP owner ($owner -> root:root)" fi fi # 2. Stock sshd — clear a start-limit lockout ONLY when the unit is genuinely `failed`. Never restart # it (unnecessary — a recreated privsep dir is picked up by the next fork; the listener never died). if systemctl is-failed --quiet "$SSHD_UNIT" 2>/dev/null; then if systemctl reset-failed "$SSHD_UNIT" 2>/dev/null; then healed=1 log "reset-failed $SSHD_UNIT (was in the failed state)" fi fi # 3. Heal marker — only on a real heal, so the agent reports the condition (hub raises a warning). if [ "$healed" = "1" ]; then date -u +%Y-%m-%dT%H:%M:%SZ > "$MARKER" 2>/dev/null || true fi exit 0