#!/bin/bash #=============================================================================== # hostinstall-mode-harness.sh — GL-2 test harness for felhom-host-install.sh's # --mode appliance|byo install profile (spec: TASK GL-2; pattern precedent: # felhom-agent/scripts/mkfs-guarded-harness.sh). # # Two tiers, PASS/FAIL per case, nonzero exit on any FAIL: # # STATIC tier (runs anywhere with bash + python3 — incl. a Windows Git Bash): # bash -n; shellcheck (if present); the argument-refusal matrix C1-C4 # (exit nonzero + the exact die message, asserted BEFORE any host access); # the C6 gate's presence; grep-invariants (exactly one GATED # step_break_glass call site, chpasswd unreachable outside its body, # --mode in usage). # # PVE tier (only on a PVE host as root; skips itself cleanly elsewhere): # C5 (byo dies naming a bogus --acl-storages entry — read-only, dies in # preflight before any hub contact) and the Scenario A/B dry-run transcript # checks. A/B need a real customer + retrieval passphrase — provide them # via FELHOM_TEST_CUSTOMER + FELHOM_TEST_PASSFILE (a 0600 passphrase file, # the script's EXISTING non-interactive path); unset -> those cases SKIP. # # STATE SAFETY (spec §9 rule 9): every invocation of the script under test runs # with FELHOM_INSTALL_STATE_DIR pointed at a throwaway temp dir, so NO case can # ever touch a live install's /var/lib/felhom-install/state.json. (Dry-run # state helpers are additionally no-ops — this is the belt on top.) # # Red-proofs (spec Part 4): run the harness against a MUTATED COPY of the # script (gate removed / requirement dropped) and watch the matching case FAIL: # RP-1 un-gate the step_break_glass call site -> "break-glass call site gated" FAILs # RP-2 drop the byo --cores/--memory requirement -> C1 FAILs # RP-3 drop the resume mode-mismatch check -> C4 FAILs # # Usage: hostinstall-mode-harness.sh [path-to-felhom-host-install.sh] #=============================================================================== set -euo pipefail SCRIPT="${1:-$(dirname "$0")/felhom-host-install.sh}" [[ -r "$SCRIPT" ]] || { echo "no script under test at $SCRIPT" >&2; exit 2; } command -v python3 >/dev/null || { echo "python3 required (the script's state helpers use it)" >&2; exit 2; } pass=0; fail=0; skip=0 verdict() { # PASS|FAIL|SKIP [detail] local v="$1" name="$2" detail="${3:-}" case "$v" in PASS) pass=$((pass+1)) ;; FAIL) fail=$((fail+1)) ;; SKIP) skip=$((skip+1)) ;; esac printf '%-4s %s\n' "$v" "$name" [[ -n "$detail" ]] && printf ' %s\n' "$detail" return 0 } # Throwaway state dir for EVERY run of the script under test (never the live one). On Git Bash the # script's python3 may be a native Windows build — hand it a Windows-syntax path via cygpath -m. WORK="$(mktemp -d "${TMPDIR:-/tmp}/hostinstall-harness.XXXXXX")" trap 'rm -rf "$WORK"' EXIT STATE_OVERRIDE="$WORK/state" mkdir -p "$STATE_OVERRIDE" if command -v cygpath >/dev/null 2>&1; then STATE_OVERRIDE_ENV="$(cygpath -m "$STATE_OVERRIDE")" else STATE_OVERRIDE_ENV="$STATE_OVERRIDE" fi # run_script — run the script under test with the state override; captures stdout+stderr # into $out and the exit code into $rc. Never lets a nonzero rc kill the harness. out=""; rc=0 run_script() { set +e out=$(FELHOM_INSTALL_STATE_DIR="$STATE_OVERRIDE_ENV" bash "$SCRIPT" "$@" 2>&1) rc=$? set -e } # expect_die -- — the script must exit nonzero AND print msg-substr. expect_die() { local name="$1" msg="$2"; shift 2 [[ "${1:-}" == "--" ]] && shift run_script "$@" if [[ $rc -ne 0 && "$out" == *"$msg"* ]]; then verdict PASS "$name" else verdict FAIL "$name" "rc=$rc; wanted substring: '$msg'; got: $(echo "$out" | tail -3 | tr '\n' ' ')" fi } echo "=== hostinstall-mode-harness — script under test: $SCRIPT ===" echo "" echo "--- STATIC tier ---" # S1: syntax if bash -n "$SCRIPT" 2>"$WORK/bashn.err"; then verdict PASS "bash -n" else verdict FAIL "bash -n" "$(cat "$WORK/bashn.err")" fi # S2: shellcheck (best-effort — required by the green gate, but the harness itself degrades) if command -v shellcheck >/dev/null 2>&1; then if shellcheck --shell=bash --severity=warning "$SCRIPT" >"$WORK/sc.out" 2>&1; then verdict PASS "shellcheck (severity>=warning)" else verdict FAIL "shellcheck (severity>=warning)" "$(head -5 "$WORK/sc.out")" fi else verdict SKIP "shellcheck" "not installed here — run it on the build server (green gate still requires it)" fi # C1: byo without --cores/--memory refuses (both missing, and each alone) expect_die "C1 byo without caps refused" \ "byo mode requires explicit --cores and --memory" \ -- --customer-id t --mode byo expect_die "C1b byo with only --cores refused" \ "byo mode requires explicit --cores and --memory" \ -- --customer-id t --mode byo --cores 4 expect_die "C1c byo with only --memory refused" \ "byo mode requires explicit --cores and --memory" \ -- --customer-id t --mode byo --memory 8192 # C2: byo refuses --enable-oob / --rotate-recovery, naming the flag expect_die "C2a byo --enable-oob refused" \ "--enable-oob is not allowed in byo mode" \ -- --customer-id t --mode byo --cores 4 --memory 8192 --enable-oob expect_die "C2b byo --rotate-recovery refused" \ "--rotate-recovery is not allowed in byo mode" \ -- --customer-id t --mode byo --cores 4 --memory 8192 --rotate-recovery # C3: fresh install without --mode refuses, naming both modes expect_die "C3 fresh install without --mode refused" \ "--mode is required: pass --mode appliance" \ -- --customer-id t expect_die "C3b unknown --mode refused" \ "Unknown --mode: bogus (appliance|byo)" \ -- --customer-id t --mode bogus expect_die "C3c retired --mode provision refused" \ "--mode provision was retired" \ -- --customer-id t --mode provision # C4: --resume with a state.json recording the OTHER mode refuses printf '{"completed":["preflight"],"customer_id":"t","mode":"appliance"}\n' > "$STATE_OVERRIDE/state.json" expect_die "C4 resume mode-mismatch refused" \ "install started as appliance; resume with --mode appliance or start over" \ -- --customer-id t --mode byo --cores 4 --memory 8192 --resume rm -f "$STATE_OVERRIDE/state.json" # C6 (static shape): the byo PVE-major gate + its message exist (a non-9.x host isn't available) if grep -q 'byo mode is validated on PVE 9.x only' "$SCRIPT" \ && grep -B3 'byo mode is validated on PVE 9.x only' "$SCRIPT" | grep -q 'MODE" == "byo"'; then verdict PASS "C6 byo PVE-major gate present (static grep)" else verdict FAIL "C6 byo PVE-major gate present (static grep)" fi # INV-1: exactly ONE step_break_glass invocation, and it is gated on MODE == appliance. # (Comment lines and the function definition itself don't count.) inv_calls=$(grep -n 'step_break_glass' "$SCRIPT" \ | grep -v -E '^[0-9]+:[[:space:]]*#' \ | grep -v 'step_break_glass()' || true) inv_count=$(echo "$inv_calls" | grep -c 'step_break_glass' || true) if [[ "$inv_count" == "1" ]] \ && echo "$inv_calls" | grep -q 'should_skip break_glass' \ && grep -B4 'should_skip break_glass' "$SCRIPT" | grep -q 'MODE" == "appliance"'; then verdict PASS "INV-1 break-glass call site: exactly one, appliance-gated" else verdict FAIL "INV-1 break-glass call site: exactly one, appliance-gated" "count=$inv_count; calls: $(echo "$inv_calls" | tr '\n' ' ')" fi # INV-2: chpasswd is unreachable outside step_break_glass's body (non-comment occurrences only). body=$(awk '/^step_break_glass\(\)/{s=NR} s && /^\}/{print s, NR; exit}' "$SCRIPT") bstart=${body% *}; bend=${body#* } inv2_ok=true while IFS=: read -r ln _; do [[ -z "$ln" ]] && continue if [[ "$ln" -lt "$bstart" || "$ln" -gt "$bend" ]]; then inv2_ok=false; fi done < <(grep -n 'chpasswd' "$SCRIPT" | grep -v -E '^[0-9]+:[[:space:]]*#' || true) if [[ -n "$bstart" && -n "$bend" ]] && $inv2_ok; then verdict PASS "INV-2 chpasswd unreachable outside step_break_glass ($bstart-$bend)" else verdict FAIL "INV-2 chpasswd unreachable outside step_break_glass" "body=$bstart-$bend" fi # INV-3: --mode is documented in usage (run the real -h path) run_script -h if [[ $rc -eq 0 && "$out" == *"--mode appliance|byo"* ]]; then verdict PASS "INV-3 usage documents --mode appliance|byo" else verdict FAIL "INV-3 usage documents --mode appliance|byo" "rc=$rc" fi echo "" echo "--- GL-4 static tier (key-pin + uninstall parity) ---" # GL4-C3: --operator-pubkey-file refusals (each dies at argv-validation, before any host access). GOODKEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeFakeFakeFakeFakeFakeFakeFakeFakeFake felhom-op-1" printf 'signer %s\n' "$GOODKEY" > "$WORK/keys-badrole" expect_die "GL4-C3a key file: unknown role refused" \ "unknown role 'signer' (want operational|recovery)" \ -- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-badrole" printf 'operational not-a-key-at-all\n' > "$WORK/keys-badline" expect_die "GL4-C3b key file: non-authorized_keys line refused" \ "not an authorized_keys line" \ -- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-badline" printf 'operational ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeFake\n' > "$WORK/keys-nocomment" expect_die "GL4-C3c key file: missing key_id comment refused" \ "no comment field" \ -- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-nocomment" printf '# only a comment\n\n' > "$WORK/keys-empty" expect_die "GL4-C3d key file: empty file refused" \ "has no key lines" \ -- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-empty" expect_die "GL4-C3e key file: missing file refused" \ "--operator-pubkey-file not found" \ -- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-nonexistent" # GL4-C2 (positive shape, runtime): a VALID key file passes resolution — the script must die LATER # (root/pveum/hub preflight, machine-dependent) and NEVER with a key-file error. printf 'operational %s\nrecovery ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeRecovery felhom-rec-1\n' "$GOODKEY" > "$WORK/keys-good" run_script --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-good" --hub-url https://127.0.0.1:9 if [[ $rc -ne 0 && "$out" != *"--operator-pubkey-file"* && "$out" != *"authorized_keys line"* && "$out" != *"unknown role"* ]]; then verdict PASS "GL4-C2 valid key file accepted (dies later, not at key parse)" else verdict FAIL "GL4-C2 valid key file accepted" "rc=$rc; $(echo "$out" | tail -2 | tr '\n' ' ')" fi # GL4-C4/C5 (grep shapes): the write-guard + override mechanics exist in the script text. if grep -q '^if signers:$' "$SCRIPT" && grep -q "replacing %d preserved authz signer" "$SCRIPT"; then verdict PASS "GL4-C4 signers-only-when-nonempty guard + preserve/replace notice present" else verdict FAIL "GL4-C4 signers-only-when-nonempty guard + preserve/replace notice present" fi if grep -q 'overrides the script.s built-in operator key constants' "$SCRIPT" \ && grep -q 'RESOLVED_OP_ID=""; RESOLVED_OP_LINE=""; RESOLVED_REC_ID=""; RESOLVED_REC_LINE=""' "$SCRIPT"; then verdict PASS "GL4-C5 file-overrides-constants mechanics present (notice + reset)" else verdict FAIL "GL4-C5 file-overrides-constants mechanics present" fi if grep -q 'no operator key pinned' "$SCRIPT" && grep -B1 'no operator key pinned' "$SCRIPT" | grep -q 'log_warn'; then verdict PASS "GL4-C1 verify dormant path is a WARN (not an error)" else verdict FAIL "GL4-C1 verify dormant path is a WARN (not an error)" fi # GL4-D: disclosure↔uninstall parity — every host artifact the byo disclosure names must be covered # (removed or explicitly KEPT) in the uninstall section (_guest_drive_note.._end of run_uninstall). ustart=$(grep -n '^_guest_drive_note()' "$SCRIPT" | cut -d: -f1) uend=$(grep -n '^# run_adopt_pool' "$SCRIPT" | cut -d: -f1) if [[ -n "$ustart" && -n "$uend" && "$ustart" -lt "$uend" ]]; then usect=$(sed -n "${ustart},${uend}p" "$SCRIPT") d_missing="" for tok in 'felhom-selfupdate-guarded' 'felhom-agent-rollback.service' 'felhom-agent-limits.conf' \ '.prev' 'felhom-mgmt-watchdog' 'felhom-privsep.conf' 'felhom-mkfs-guarded' \ 'felhom-guest-hook' '/mnt/felhom-drives' 'AGENT_SUDOERS' 'AGENT_STATE_DIR' \ 'remove_scoped_acl' 'pveum user token remove' 'pveum pool delete' 'STATE_FILE'; do echo "$usect" | grep -qF "$tok" || d_missing+="$tok " done if [[ -z "$d_missing" ]]; then verdict PASS "GL4-D disclosure↔uninstall parity (all artifact tokens covered)" else verdict FAIL "GL4-D disclosure↔uninstall parity" "uncovered: $d_missing" fi else verdict FAIL "GL4-D disclosure↔uninstall parity" "could not locate the uninstall section" fi # GL4-INV: no forced/lazy unmount and no format op on the drives root — REAL invocations only # (comment lines and log_* guidance strings legitimately SAY "never umount -l/-f"). if ! grep -vE '^[[:space:]]*#|log_(warn|info|dry|error|success|skip)' "$SCRIPT" | grep -E 'umount +-(l|f)' >/dev/null \ && ! grep -vE '^[[:space:]]*#' "$SCRIPT" | grep -E '(mkfs|wipefs) [^|]*/mnt/felhom-drives' >/dev/null; then verdict PASS "GL4-INV no umount -l/-f, no mkfs/wipefs invocation on /mnt/felhom-drives" else verdict FAIL "GL4-INV no umount -l/-f, no mkfs/wipefs invocation on /mnt/felhom-drives" fi echo "" echo "--- PVE tier ---" if ! command -v pveum >/dev/null 2>&1 || [[ "$(id -u)" != 0 ]]; then verdict SKIP "PVE tier (C5 + Scenario A/B dry transcripts)" "needs a PVE host as root — run there" else # C5: byo with a bogus --acl-storages entry dies in preflight NAMING it (read-only: dies before # any hub contact/passphrase and before any mutation; state override active). expect_die "C5 byo bogus --acl-storages refused, named" \ "acl storage(s) not found on this box: definitely-not-a-storage" \ -- --customer-id t --mode byo --cores 4 --memory 8192 \ --acl-storages "local definitely-not-a-storage" --dry-run # GL4 H-U: FULL uninstall dry transcript (Scenario A). Read-only: every mutation is dry-printed, # the typed confirm takes its dry branch, and the state override keeps the live state.json out. # Requires a Felhom guest to target — resolved from felhom_guests-style detection below. hu_vmid=$(for id in $( { pct list 2>/dev/null; qm list 2>/dev/null; } | awk "{print \$1}" | grep -E "^[0-9]+$" ); do pct config "$id" 2>/dev/null | grep -q "mp=/etc/felhom-bootstrap" && { echo "$id"; break; } done) if [[ -z "$hu_vmid" ]]; then verdict SKIP "GL4 H-U uninstall dry transcript" "no Felhom guest on this host to target" else run_script --uninstall --vmid "$hu_vmid" --dry-run hu_ok=true; hu_why="" [[ $rc -eq 0 ]] || { hu_ok=false; hu_why+="rc=$rc "; } for want in "felhom-selfupdate-guarded" "kept vs wiped" "NEVER wiped"; do [[ "$out" == *"$want"* ]] || { hu_ok=false; hu_why+="missing '$want' "; } done # MUST NOT: forced/lazy unmounts, or ANY destructive op (mkfs/wipefs/rm) on a drive-data # path. Removing the felhom-mkfs-guarded WRAPPER from /usr/local/sbin is legitimate. for bad in "umount -l" "umount -f"; do [[ "$out" != *"$bad"* ]] || { hu_ok=false; hu_why+="contains '$bad' "; } done if echo "$out" | grep -E '(mkfs|wipefs|rm |rm -rf).*/mnt/felhom-drives/' >/dev/null; then hu_ok=false; hu_why+="destructive op on a /mnt/felhom-drives/ path " fi # drive umount lines only when child mounts exist — assert conditionally if findmnt -rn -o TARGET 2>/dev/null | grep -q '^/mnt/felhom-drives/'; then [[ "$out" == *"data stays on the drive"* ]] || { hu_ok=false; hu_why+="missing per-drive umount lines "; } fi if $hu_ok; then verdict PASS "GL4 H-U uninstall dry transcript (guest $hu_vmid; statement + selfupdate removal, no forced ops)" else verdict FAIL "GL4 H-U uninstall dry transcript" "$hu_why" fi fi if [[ -n "${FELHOM_TEST_CUSTOMER:-}" && -n "${FELHOM_TEST_PASSFILE:-}" && -r "${FELHOM_TEST_PASSFILE:-}" ]]; then common=(--customer-id "$FELHOM_TEST_CUSTOMER" --passphrase-file "$FELHOM_TEST_PASSFILE" \ --vmid 990100 --cores 2 --memory 4096 --dry-run) # H-A: appliance dry transcript still walks 4b/8 (regression guard) run_script --mode appliance "${common[@]}" if [[ $rc -eq 0 && "$out" == *"4b/8"* ]]; then verdict PASS "H-A appliance dry transcript contains 4b/8" else verdict FAIL "H-A appliance dry transcript contains 4b/8" "rc=$rc" fi # H-B: byo dry transcript — no 4b/8 / chpasswd / recovery-credential; ack + caps + storages present run_script --mode byo "${common[@]}" hb_ok=true; hb_why="" [[ $rc -eq 0 ]] || { hb_ok=false; hb_why+="rc=$rc "; } for bad in "4b/8" "chpasswd" "recovery-credential"; do [[ "$out" != *"$bad"* ]] || { hb_ok=false; hb_why+="contains '$bad' "; } done for want in "acknowledge the byo install" "acl storages all present" "-cores 2 -memory 4096" "self-update authority"; do [[ "$out" == *"$want"* ]] || { hb_ok=false; hb_why+="missing '$want' "; } done if $hb_ok; then verdict PASS "H-B byo dry transcript (no root@pam path; ack+caps+storages present)" else verdict FAIL "H-B byo dry transcript" "$hb_why" fi else verdict SKIP "H-A/H-B dry transcripts" "set FELHOM_TEST_CUSTOMER + FELHOM_TEST_PASSFILE (0600 passphrase file) to run" fi fi echo "" echo "=== $pass passed, $fail failed, $skip skipped ===" [[ $fail -eq 0 ]]