admin
88aecf55b6
gates / gates (push) Successful in 7m23s
Live on demo-hp (--rotate-recovery-only): vault time moved, new value 200, old 401. Harness 52/0; the R1145 cases fail against HEAD. Publishes only with the next installer tag.
653 lines
35 KiB
Bash
653 lines
35 KiB
Bash
#!/bin/bash
|
|
#===============================================================================
|
|
# hostinstall-mode-harness.sh — GL-2 test harness for felhom-host-install.sh's
|
|
# --mode appliance|byo install profile (spec: TASK GL-2; pattern precedent:
|
|
# felhom-agent/scripts/mkfs-guarded-harness.sh).
|
|
#
|
|
# Two tiers, PASS/FAIL per case, nonzero exit on any FAIL:
|
|
#
|
|
# STATIC tier (runs anywhere with bash + python3 — incl. a Windows Git Bash):
|
|
# bash -n; shellcheck (if present); the argument-refusal matrix C1-C4
|
|
# (exit nonzero + the exact die message, asserted BEFORE any host access);
|
|
# the C6 gate's presence; grep-invariants (exactly one GATED
|
|
# step_break_glass call site, chpasswd unreachable outside its body,
|
|
# --mode in usage).
|
|
#
|
|
# PVE tier (only on a PVE host as root; skips itself cleanly elsewhere):
|
|
# C5 (byo dies naming a bogus --acl-storages entry — read-only, dies in
|
|
# preflight before any hub contact) and the Scenario A/B dry-run transcript
|
|
# checks. A/B need a real customer + retrieval passphrase — provide them
|
|
# via FELHOM_TEST_CUSTOMER + FELHOM_TEST_PASSFILE (a 0600 passphrase file,
|
|
# the script's EXISTING non-interactive path); unset -> those cases SKIP.
|
|
#
|
|
# STATE SAFETY (spec §9 rule 9): every invocation of the script under test runs
|
|
# with FELHOM_INSTALL_STATE_DIR pointed at a throwaway temp dir, so NO case can
|
|
# ever touch a live install's /var/lib/felhom-install/state.json. (Dry-run
|
|
# state helpers are additionally no-ops — this is the belt on top.)
|
|
#
|
|
# Red-proofs (spec Part 4): run the harness against a MUTATED COPY of the
|
|
# script (gate removed / requirement dropped) and watch the matching case FAIL:
|
|
# RP-1 un-gate the step_break_glass call site -> "break-glass call site gated" FAILs
|
|
# RP-2 drop the byo --cores/--memory requirement -> C1 FAILs
|
|
# RP-3 drop the resume mode-mismatch check -> C4 FAILs
|
|
#
|
|
# Usage: hostinstall-mode-harness.sh [path-to-felhom-host-install.sh]
|
|
#===============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT="${1:-$(dirname "$0")/felhom-host-install.sh}"
|
|
[[ -r "$SCRIPT" ]] || { echo "no script under test at $SCRIPT" >&2; exit 2; }
|
|
command -v python3 >/dev/null || { echo "python3 required (the script's state helpers use it)" >&2; exit 2; }
|
|
|
|
pass=0; fail=0; skip=0
|
|
verdict() { # PASS|FAIL|SKIP <name> [detail]
|
|
local v="$1" name="$2" detail="${3:-}"
|
|
case "$v" in
|
|
PASS) pass=$((pass+1)) ;;
|
|
FAIL) fail=$((fail+1)) ;;
|
|
SKIP) skip=$((skip+1)) ;;
|
|
esac
|
|
printf '%-4s %s\n' "$v" "$name"
|
|
[[ -n "$detail" ]] && printf ' %s\n' "$detail"
|
|
return 0
|
|
}
|
|
|
|
# Throwaway state dir for EVERY run of the script under test (never the live one). On Git Bash the
|
|
# script's python3 may be a native Windows build — hand it a Windows-syntax path via cygpath -m.
|
|
WORK="$(mktemp -d "${TMPDIR:-/tmp}/hostinstall-harness.XXXXXX")"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
STATE_OVERRIDE="$WORK/state"
|
|
mkdir -p "$STATE_OVERRIDE"
|
|
if command -v cygpath >/dev/null 2>&1; then
|
|
STATE_OVERRIDE_ENV="$(cygpath -m "$STATE_OVERRIDE")"
|
|
else
|
|
STATE_OVERRIDE_ENV="$STATE_OVERRIDE"
|
|
fi
|
|
|
|
# run_script <args...> — run the script under test with the state override; captures stdout+stderr
|
|
# into $out and the exit code into $rc. Never lets a nonzero rc kill the harness.
|
|
out=""; rc=0
|
|
run_script() {
|
|
set +e
|
|
out=$(FELHOM_INSTALL_STATE_DIR="$STATE_OVERRIDE_ENV" bash "$SCRIPT" "$@" 2>&1)
|
|
rc=$?
|
|
set -e
|
|
}
|
|
|
|
# expect_die <name> <msg-substr> -- <args...> — the script must exit nonzero AND print msg-substr.
|
|
expect_die() {
|
|
local name="$1" msg="$2"; shift 2
|
|
[[ "${1:-}" == "--" ]] && shift
|
|
run_script "$@"
|
|
if [[ $rc -ne 0 && "$out" == *"$msg"* ]]; then
|
|
verdict PASS "$name"
|
|
else
|
|
verdict FAIL "$name" "rc=$rc; wanted substring: '$msg'; got: $(echo "$out" | tail -3 | tr '\n' ' ')"
|
|
fi
|
|
}
|
|
|
|
echo "=== hostinstall-mode-harness — script under test: $SCRIPT ==="
|
|
echo ""
|
|
echo "--- STATIC tier ---"
|
|
|
|
# S1: syntax
|
|
if bash -n "$SCRIPT" 2>"$WORK/bashn.err"; then
|
|
verdict PASS "bash -n"
|
|
else
|
|
verdict FAIL "bash -n" "$(cat "$WORK/bashn.err")"
|
|
fi
|
|
|
|
# S2: shellcheck (best-effort — required by the green gate, but the harness itself degrades)
|
|
if command -v shellcheck >/dev/null 2>&1; then
|
|
if shellcheck --shell=bash --severity=warning "$SCRIPT" >"$WORK/sc.out" 2>&1; then
|
|
verdict PASS "shellcheck (severity>=warning)"
|
|
else
|
|
verdict FAIL "shellcheck (severity>=warning)" "$(head -5 "$WORK/sc.out")"
|
|
fi
|
|
else
|
|
verdict SKIP "shellcheck" "not installed here — run it on the build server (green gate still requires it)"
|
|
fi
|
|
|
|
# C1: byo without --cores/--memory refuses (both missing, and each alone)
|
|
expect_die "C1 byo without caps refused" \
|
|
"byo mode requires explicit --cores and --memory" \
|
|
-- --customer-id t --mode byo
|
|
expect_die "C1b byo with only --cores refused" \
|
|
"byo mode requires explicit --cores and --memory" \
|
|
-- --customer-id t --mode byo --cores 4
|
|
expect_die "C1c byo with only --memory refused" \
|
|
"byo mode requires explicit --cores and --memory" \
|
|
-- --customer-id t --mode byo --memory 8192
|
|
|
|
# C2: byo refuses --enable-oob / --rotate-recovery, naming the flag
|
|
expect_die "C2a byo --enable-oob refused" \
|
|
"--enable-oob is not allowed in byo mode" \
|
|
-- --customer-id t --mode byo --cores 4 --memory 8192 --enable-oob
|
|
expect_die "C2b byo --rotate-recovery refused" \
|
|
"--rotate-recovery is not allowed in byo mode" \
|
|
-- --customer-id t --mode byo --cores 4 --memory 8192 --rotate-recovery
|
|
|
|
# C3: fresh install without --mode refuses, naming both modes
|
|
expect_die "C3 fresh install without --mode refused" \
|
|
"--mode is required: pass --mode appliance" \
|
|
-- --customer-id t
|
|
expect_die "C3b unknown --mode refused" \
|
|
"Unknown --mode: bogus (appliance|byo)" \
|
|
-- --customer-id t --mode bogus
|
|
expect_die "C3c retired --mode provision refused" \
|
|
"--mode provision was retired" \
|
|
-- --customer-id t --mode provision
|
|
|
|
# C4: --resume with a state.json recording the OTHER mode refuses
|
|
printf '{"completed":["preflight"],"customer_id":"t","mode":"appliance"}\n' > "$STATE_OVERRIDE/state.json"
|
|
expect_die "C4 resume mode-mismatch refused" \
|
|
"install started as appliance; resume with --mode appliance or start over" \
|
|
-- --customer-id t --mode byo --cores 4 --memory 8192 --resume
|
|
rm -f "$STATE_OVERRIDE/state.json"
|
|
|
|
# C6 (static shape): the byo PVE-major gate + its message exist (a non-9.x host isn't available)
|
|
if grep -q 'byo mode is validated on PVE 9.x only' "$SCRIPT" \
|
|
&& grep -B3 'byo mode is validated on PVE 9.x only' "$SCRIPT" | grep -q 'MODE" == "byo"'; then
|
|
verdict PASS "C6 byo PVE-major gate present (static grep)"
|
|
else
|
|
verdict FAIL "C6 byo PVE-major gate present (static grep)"
|
|
fi
|
|
|
|
# INV-1: exactly TWO step_break_glass invocations, each appliance-gated: the install's (gated on
|
|
# MODE == appliance) and --rotate-recovery-only's (R-1141, gated on the agent config's
|
|
# deployment_mode == appliance). (Comment lines and the function definition itself don't count.)
|
|
inv_calls=$(grep -n 'step_break_glass' "$SCRIPT" \
|
|
| grep -v -E '^[0-9]+:[[:space:]]*#' \
|
|
| grep -v 'step_break_glass()' || true)
|
|
inv_count=$(echo "$inv_calls" | grep -c 'step_break_glass' || true)
|
|
inv_bare=$(echo "$inv_calls" | grep -v 'should_skip break_glass' | cut -d: -f1 || true)
|
|
if [[ "$inv_count" == "2" ]] \
|
|
&& echo "$inv_calls" | grep -q 'should_skip break_glass' \
|
|
&& grep -B4 'should_skip break_glass' "$SCRIPT" | grep -q 'MODE" == "appliance"' \
|
|
&& [[ -n "$inv_bare" ]] \
|
|
&& sed -n "$((inv_bare-1))p" "$SCRIPT" | grep -q 'ROTATE_CFG_MODE" == "appliance"'; then
|
|
verdict PASS "INV-1 break-glass call sites: exactly two, both appliance-gated"
|
|
else
|
|
verdict FAIL "INV-1 break-glass call sites: exactly two, both appliance-gated" "count=$inv_count; calls: $(echo "$inv_calls" | tr '\n' ' ')"
|
|
fi
|
|
|
|
# INV-2: chpasswd is unreachable outside step_break_glass's body (non-comment occurrences only).
|
|
body=$(awk '/^step_break_glass\(\)/{s=NR} s && /^\}/{print s, NR; exit}' "$SCRIPT")
|
|
bstart=${body% *}; bend=${body#* }
|
|
inv2_ok=true
|
|
while IFS=: read -r ln _; do
|
|
[[ -z "$ln" ]] && continue
|
|
if [[ "$ln" -lt "$bstart" || "$ln" -gt "$bend" ]]; then inv2_ok=false; fi
|
|
done < <(grep -n 'chpasswd' "$SCRIPT" | grep -v -E '^[0-9]+:[[:space:]]*#' || true)
|
|
if [[ -n "$bstart" && -n "$bend" ]] && $inv2_ok; then
|
|
verdict PASS "INV-2 chpasswd unreachable outside step_break_glass ($bstart-$bend)"
|
|
else
|
|
verdict FAIL "INV-2 chpasswd unreachable outside step_break_glass" "body=$bstart-$bend"
|
|
fi
|
|
|
|
# INV-3: --mode is documented in usage (run the real -h path)
|
|
run_script -h
|
|
if [[ $rc -eq 0 && "$out" == *"--mode appliance|byo"* ]]; then
|
|
verdict PASS "INV-3 usage documents --mode appliance|byo"
|
|
else
|
|
verdict FAIL "INV-3 usage documents --mode appliance|byo" "rc=$rc"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- GL-4 static tier (key-pin + uninstall parity) ---"
|
|
|
|
# GL4-C3: --operator-pubkey-file refusals (each dies at argv-validation, before any host access).
|
|
GOODKEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeFakeFakeFakeFakeFakeFakeFakeFakeFake felhom-op-1"
|
|
printf 'signer %s\n' "$GOODKEY" > "$WORK/keys-badrole"
|
|
expect_die "GL4-C3a key file: unknown role refused" \
|
|
"unknown role 'signer' (want operational|recovery)" \
|
|
-- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-badrole"
|
|
printf 'operational not-a-key-at-all\n' > "$WORK/keys-badline"
|
|
expect_die "GL4-C3b key file: non-authorized_keys line refused" \
|
|
"not an authorized_keys line" \
|
|
-- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-badline"
|
|
printf 'operational ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeFake\n' > "$WORK/keys-nocomment"
|
|
expect_die "GL4-C3c key file: missing key_id comment refused" \
|
|
"no comment field" \
|
|
-- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-nocomment"
|
|
printf '# only a comment\n\n' > "$WORK/keys-empty"
|
|
expect_die "GL4-C3d key file: empty file refused" \
|
|
"has no key lines" \
|
|
-- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-empty"
|
|
expect_die "GL4-C3e key file: missing file refused" \
|
|
"--operator-pubkey-file not found" \
|
|
-- --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-nonexistent"
|
|
|
|
# GL4-C2 (positive shape, runtime): a VALID key file passes resolution — the script must die LATER
|
|
# (root/pveum/hub preflight, machine-dependent) and NEVER with a key-file ERROR. Since v1.11.1 the
|
|
# constants are pinned, so the benign "file overrides constants" NOTICE legitimately appears —
|
|
# assert on the actual die messages, not on any mention of the flag.
|
|
printf 'operational %s\nrecovery ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeFakeRecovery felhom-rec-1\n' "$GOODKEY" > "$WORK/keys-good"
|
|
run_script --customer-id t --mode appliance --operator-pubkey-file "$WORK/keys-good" --hub-url https://127.0.0.1:9
|
|
if [[ $rc -ne 0 && "$out" != *"authorized_keys line"* && "$out" != *"unknown role"* \
|
|
&& "$out" != *"has no key lines"* && "$out" != *"key line has no comment"* \
|
|
&& "$out" != *"--operator-pubkey-file not found"* ]]; then
|
|
verdict PASS "GL4-C2 valid key file accepted (dies later, not at key parse)"
|
|
else
|
|
verdict FAIL "GL4-C2 valid key file accepted" "rc=$rc; $(echo "$out" | tail -2 | tr '\n' ' ')"
|
|
fi
|
|
|
|
# GL4-C4/C5 (grep shapes): the write-guard + override mechanics exist in the script text.
|
|
if grep -q '^if signers:$' "$SCRIPT" && grep -q "replacing %d preserved authz signer" "$SCRIPT"; then
|
|
verdict PASS "GL4-C4 signers-only-when-nonempty guard + preserve/replace notice present"
|
|
else
|
|
verdict FAIL "GL4-C4 signers-only-when-nonempty guard + preserve/replace notice present"
|
|
fi
|
|
if grep -q 'overrides the script.s built-in operator key constants' "$SCRIPT" \
|
|
&& grep -q 'RESOLVED_OP_ID=""; RESOLVED_OP_LINE=""; RESOLVED_REC_ID=""; RESOLVED_REC_LINE=""' "$SCRIPT"; then
|
|
verdict PASS "GL4-C5 file-overrides-constants mechanics present (notice + reset)"
|
|
else
|
|
verdict FAIL "GL4-C5 file-overrides-constants mechanics present"
|
|
fi
|
|
if grep -q 'no operator key pinned' "$SCRIPT" && grep -B1 'no operator key pinned' "$SCRIPT" | grep -q 'log_warn'; then
|
|
verdict PASS "GL4-C1 verify dormant path is a WARN (not an error)"
|
|
else
|
|
verdict FAIL "GL4-C1 verify dormant path is a WARN (not an error)"
|
|
fi
|
|
|
|
# GL4-D: disclosure↔uninstall parity — every host artifact the byo disclosure names must be covered
|
|
# (removed or explicitly KEPT) in the uninstall section (_guest_drive_note.._end of run_uninstall).
|
|
ustart=$(grep -n '^_guest_drive_note()' "$SCRIPT" | cut -d: -f1)
|
|
uend=$(grep -n '^# run_adopt_pool' "$SCRIPT" | cut -d: -f1)
|
|
if [[ -n "$ustart" && -n "$uend" && "$ustart" -lt "$uend" ]]; then
|
|
usect=$(sed -n "${ustart},${uend}p" "$SCRIPT")
|
|
d_missing=""
|
|
for tok in 'felhom-selfupdate-guarded' 'felhom-agent-rollback.service' 'felhom-agent-limits.conf' \
|
|
'.prev' 'felhom-mgmt-watchdog' 'felhom-privsep.conf' 'felhom-mkfs-guarded' \
|
|
'felhom-guest-hook' '/mnt/felhom-drives' 'AGENT_SUDOERS' 'AGENT_STATE_DIR' \
|
|
'remove_scoped_acl' 'pveum user token remove' 'pveum pool delete' 'STATE_FILE' \
|
|
'felhom-priv-apply' '_teardown_wg_tunnel' '_purge_agent_config'; do
|
|
# a here-string, not `echo | grep -q`: under pipefail grep -q's early exit SIGPIPEs the echo
|
|
# and a token that IS present reads as missing (AGENT_SUDOERS did, 2026-10-05).
|
|
grep -qF -- "$tok" <<<"$usect" || d_missing+="$tok "
|
|
done
|
|
if [[ -z "$d_missing" ]]; then
|
|
verdict PASS "GL4-D disclosure↔uninstall parity (all artifact tokens covered)"
|
|
else
|
|
verdict FAIL "GL4-D disclosure↔uninstall parity" "uncovered: $d_missing"
|
|
fi
|
|
else
|
|
verdict FAIL "GL4-D disclosure↔uninstall parity" "could not locate the uninstall section"
|
|
fi
|
|
|
|
# GL6-ANON (v1.11.2, the Gate-0 operator ruling): empty git creds warn + fetch anonymously — the
|
|
# resolve must NOT die on an empty token, and every artifact fetch must build its auth args
|
|
# conditionally (a curl -u with an empty token would 401 even on world-readable content).
|
|
if grep -q 'fetching artifacts ANONYMOUSLY' "$SCRIPT" \
|
|
&& grep -B1 'fetching artifacts ANONYMOUSLY' "$SCRIPT" | grep -q 'log_warn' \
|
|
&& ! grep -q 'no git token in controller.yaml — cannot fetch' "$SCRIPT" \
|
|
&& [ "$(grep -c '_git_auth_args _auth' "$SCRIPT")" -ge 2 ] \
|
|
&& ! grep -E 'curl -fsS -u "\$\{GIT_USER\}' "$SCRIPT" >/dev/null; then
|
|
verdict PASS "GL6-ANON empty-cred anonymous-fetch fallback (warn-not-die + conditional auth)"
|
|
else
|
|
verdict FAIL "GL6-ANON empty-cred anonymous-fetch fallback (warn-not-die + conditional auth)"
|
|
fi
|
|
|
|
# GL6-F4 (v1.11.3): the producer steps token/enroll/grows must run UNCONDITIONALLY (not behind
|
|
# `should_skip … ||`) so --resume repopulates the in-memory outputs later steps consume; and the
|
|
# resume block must re-derive GOLDEN_VOLID from the local archive when golden already completed.
|
|
if grep -qE '^step_token$' "$SCRIPT" && grep -qE '^step_grows$' "$SCRIPT" && grep -qE '^step_enroll$' "$SCRIPT" \
|
|
&& ! grep -qE 'should_skip (token|grows|enroll) +\|\| +step_' "$SCRIPT" \
|
|
&& grep -q 'golden (resumed from local)' "$SCRIPT"; then
|
|
verdict PASS "GL6-F4 resume repopulates producer outputs (token/enroll/grows unconditional + golden re-derive)"
|
|
else
|
|
verdict FAIL "GL6-F4 resume repopulates producer outputs (token/enroll/grows unconditional + golden re-derive)"
|
|
fi
|
|
|
|
# GL4-INV: no forced/lazy unmount and no format op on the drives root — REAL invocations only
|
|
# (comment lines and log_* guidance strings legitimately SAY "never umount -l/-f").
|
|
if ! grep -vE '^[[:space:]]*#|log_(warn|info|dry|error|success|skip)' "$SCRIPT" | grep -E 'umount +-(l|f)' >/dev/null \
|
|
&& ! grep -vE '^[[:space:]]*#' "$SCRIPT" | grep -E '(mkfs|wipefs) [^|]*/mnt/felhom-drives' >/dev/null; then
|
|
verdict PASS "GL4-INV no umount -l/-f, no mkfs/wipefs invocation on /mnt/felhom-drives"
|
|
else
|
|
verdict FAIL "GL4-INV no umount -l/-f, no mkfs/wipefs invocation on /mnt/felhom-drives"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- GL-8 static tier (BYO coexistence hardening) ---"
|
|
|
|
# GL8-F3: the populated-host leaf guard tests for a FELHOM guest (felhom_guests), NOT any guest
|
|
# (pct list). Runtime behaviour is impractical to isolate (the guard is inline in step 5, gated on
|
|
# the real state dir's leaf) — validated grep-level + red-proof here; the true behavioural proof is
|
|
# GL-7's real byo install on a populated non-Felhom host.
|
|
if grep -A2 'this host already has a Felhom guest' "$SCRIPT" >/dev/null 2>&1 \
|
|
&& grep -B3 'this host already has a Felhom guest' "$SCRIPT" | grep -q '\[\[ -n "\$(felhom_guests)" \]\]' \
|
|
&& ! grep -B3 'has no agent leaf to preserve' "$SCRIPT" | grep -q 'pct list .* | .*tail -n +2 | grep -q'; then
|
|
verdict PASS "GL8-F3 leaf guard uses felhom_guests (not pct list — fires on Felhom guests only)"
|
|
else
|
|
verdict FAIL "GL8-F3 leaf guard uses felhom_guests (not pct list — fires on Felhom guests only)"
|
|
fi
|
|
|
|
# GL8-F6: the byo :53 gate REFUSES (dies) with the remediation and mutates nothing — no
|
|
# stop/mask/kill of a :53 service anywhere in the script.
|
|
# R-1146 (2026-10-18): the check is judged by WHERE a line is, not by a fixed window. The gate's R-300 hint grew the
|
|
# byo `if` beyond 6 lines above the refusal, and R-300 made the UNINSTALL turn off Felhom's OWN dnsmasq on purpose.
|
|
# Allowed: a disable inside run_uninstall (ours, R-300) or inside the gate's _hint="…" text (an instruction to the
|
|
# owner, never executed). Any other stop/mask/disable/kill of a resolver, anywhere, still FAILS.
|
|
gl8_f6_offenders=$(awk '
|
|
/^[a-zA-Z_]+\(\) *\{/ { fn = $1 }
|
|
/_hint="/ { inhint = 1 }
|
|
!/^[[:space:]]*#/ && /(systemctl (stop|mask|disable)|kill|pkill).*(dnsmasq|:53|resolv)/ {
|
|
if (fn != "run_uninstall()" && !inhint) print NR": "$0
|
|
}
|
|
inhint && /"[[:space:]]*$/ && !/_hint="[[:space:]]*$/ { inhint = 0 }
|
|
' "$SCRIPT")
|
|
gl8_f6_byo_before=$(awk '
|
|
/if \[\[ "\$MODE" == "byo" \]\]; then/ { last = NR }
|
|
/die "a resolver is already bound to :53 on this host — Felhom needs/ { print (last && NR - last <= 40) ? "yes" : "no"; exit }
|
|
' "$SCRIPT")
|
|
if grep -q 'a resolver is already bound to :53 on this host — Felhom needs the guest reachable' "$SCRIPT" \
|
|
&& [[ "$gl8_f6_byo_before" == "yes" ]] && [[ -z "$gl8_f6_offenders" ]]; then
|
|
verdict PASS "GL8-F6 byo :53 gate refuses+instructs, never mutates the owner's resolver"
|
|
else
|
|
verdict FAIL "GL8-F6 byo :53 gate refuses+instructs, never mutates the owner's resolver" "byo-gate=${gl8_f6_byo_before:-none} offenders=${gl8_f6_offenders:-none}"
|
|
fi
|
|
|
|
# GL8-F1 / R-275: the agent config and EVERY copy of it go — the agent's own config dir is purged as a
|
|
# directory (the old `.bak*` glob missed `agent.json.campaign9-before` & co.). Behaviour is pinned by
|
|
# scripts/test_hostinstall.py (test_purge_*, run in CI); here only the call site.
|
|
if grep -q '^ _purge_agent_config "\$agent_cfg"' "$SCRIPT" && ! grep -q '"${agent_cfg}".bak\*' "$SCRIPT"; then
|
|
verdict PASS "GL8-F1 uninstall purges the agent config dir (every copy, R-275)"
|
|
else
|
|
verdict FAIL "GL8-F1 uninstall purges the agent config dir (every copy, R-275)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- R-1141 --rotate-recovery-only (step 4b alone) ---"
|
|
# Runs anywhere: refusals die before any host access; the dry-run reads a FAKE agent config through the
|
|
# test-only FELHOM_AGENT_CONFIG override and points the hub at a closed port. A chpasswd/systemctl/pct
|
|
# PATH shim records any call — a dry run must make none of them.
|
|
RK="FAKE-HOST-KEY-r1141-never-printed"
|
|
mkdir -p "$WORK/r1141shim"
|
|
for c in chpasswd pct qm pveum systemctl apt-get install; do
|
|
printf '#!/bin/sh\necho "%s $*" >> "%s/r1141shim/calls"\nexit 1\n' "$c" "$WORK" > "$WORK/r1141shim/$c"
|
|
chmod 0755 "$WORK/r1141shim/$c"
|
|
done
|
|
r1141_cfg() { # <file> <deployment_mode|-> <api_key>
|
|
python3 - "$1" "$2" "$3" <<'PY'
|
|
import json, sys
|
|
f, mode, key = sys.argv[1:]
|
|
d = {"hub": {"url": "http://127.0.0.1:9", "host_id": "h-test", "api_key": key}}
|
|
if mode != "-":
|
|
d["deployment_mode"] = mode
|
|
json.dump(d, open(f, "w"))
|
|
PY
|
|
chmod 0600 "$1"
|
|
}
|
|
r1141_run() { # <config> <args...>
|
|
local cfg="$1"; shift
|
|
set +e
|
|
out=$(FELHOM_INSTALL_STATE_DIR="$STATE_OVERRIDE_ENV" FELHOM_AGENT_CONFIG="$cfg" PATH="$WORK/r1141shim:$PATH" \
|
|
bash "$SCRIPT" --rotate-recovery-only "$@" </dev/null 2>&1)
|
|
rc=$?
|
|
set -e
|
|
}
|
|
r1141_expect_die() { # <name> <msg> <config> <args...>
|
|
local name="$1" msg="$2" cfg="$3"; shift 3
|
|
r1141_run "$cfg" "$@"
|
|
if [[ $rc -ne 0 && "$out" == *"$msg"* && "$out" != *"4b/8"* ]]; then
|
|
verdict PASS "$name"
|
|
else
|
|
verdict FAIL "$name" "rc=$rc; wanted '$msg', no 4b/8; got: $(echo "$out" | tail -3 | tr '\n' ' ')"
|
|
fi
|
|
}
|
|
r1141_cfg "$WORK/r1141-app.json" appliance "$RK"
|
|
r1141_cfg "$WORK/r1141-byo.json" byo "$RK"
|
|
r1141_cfg "$WORK/r1141-nomode.json" - "$RK"
|
|
r1141_cfg "$WORK/r1141-nokey.json" appliance ""
|
|
printf 'not json' > "$WORK/r1141-bad.json"
|
|
r1141_expect_die "R1141-a --mode byo refused" "is not allowed in byo mode" "$WORK/r1141-app.json" --mode byo --dry-run
|
|
r1141_expect_die "R1141-b with --uninstall refused" "--rotate-recovery-only runs alone" "$WORK/r1141-app.json" --uninstall --dry-run
|
|
r1141_expect_die "R1141-c with --resume refused" "--rotate-recovery-only runs alone" "$WORK/r1141-app.json" --resume --dry-run
|
|
r1141_expect_die "R1141-d config deployment_mode=byo refused" "deployment_mode=byo, not appliance" "$WORK/r1141-byo.json" --dry-run
|
|
r1141_expect_die "R1141-e config without deployment_mode refused (fail-safe)" "deployment_mode=<absent>, not appliance" "$WORK/r1141-nomode.json" --dry-run
|
|
r1141_expect_die "R1141-f config without a host key refused" "no hub.host_id/hub.api_key" "$WORK/r1141-nokey.json" --dry-run
|
|
r1141_expect_die "R1141-g config not JSON refused" "agent config is not valid JSON" "$WORK/r1141-bad.json" --dry-run
|
|
r1141_expect_die "R1141-h config missing refused" "agent config not readable" "$WORK/r1141-absent.json" --dry-run
|
|
if [[ "$(id -u)" != 0 ]]; then
|
|
r1141_expect_die "R1141-i a real run needs root" "must run as root" "$WORK/r1141-app.json"
|
|
else
|
|
verdict SKIP "R1141-i a real run needs root" "running as root — the refusal cannot be shown here"
|
|
fi
|
|
# The positive dry run: step 4b's dry line only, the host key never printed, no other step, no host
|
|
# command, no state written.
|
|
rm -f "$WORK/r1141shim/calls" "$STATE_OVERRIDE/state.json"
|
|
r1141_run "$WORK/r1141-app.json" --dry-run
|
|
r_ok=true; r_why=""
|
|
[[ $rc -eq 0 ]] || { r_ok=false; r_why+="rc=$rc "; }
|
|
for want in "mode=rotate-recovery-only" "4b/8 break-glass credential" \
|
|
"PUT http://127.0.0.1:9/api/v1/hosts/h-test/recovery-credential" "a real run would stop here"; do
|
|
[[ "$out" == *"$want"* ]] || { r_ok=false; r_why+="missing '$want' "; }
|
|
done
|
|
for bad in "$RK" "1/8" "2/8" "3/8" "4/8" "5/8" "6/8" "7/8" "8/8" "agent install" "PASSPHRASE" "retrieval passphrase"; do
|
|
[[ "$out" != *"$bad"* ]] || { r_ok=false; r_why+="contains '$bad' "; }
|
|
done
|
|
[[ ! -s "$WORK/r1141shim/calls" ]] || { r_ok=false; r_why+="host commands called: $(tr '\n' ';' < "$WORK/r1141shim/calls") "; }
|
|
[[ ! -e "$STATE_OVERRIDE/state.json" ]] || { r_ok=false; r_why+="state.json written "; }
|
|
if $r_ok; then
|
|
verdict PASS "R1141-j dry run: step 4b only, host key not printed, no host command, no state"
|
|
else
|
|
verdict FAIL "R1141-j dry run: step 4b only, host key not printed, no host command, no state" "$r_why"
|
|
fi
|
|
# Usage documents the flag (the real -h path).
|
|
run_script -h
|
|
if [[ $rc -eq 0 && "$out" == *"--rotate-recovery-only"* ]]; then
|
|
verdict PASS "R1141-k usage documents --rotate-recovery-only"
|
|
else
|
|
verdict FAIL "R1141-k usage documents --rotate-recovery-only" "rc=$rc"
|
|
fi
|
|
|
|
# ── R-1145 (2026-10-18): step 4b is VAULT FIRST, THEN SET. The REAL function (cut out of the script) runs with fake
|
|
# openssl/curl/chpasswd/sleep on PATH and a fake shadow file; each fake appends one word to a calls file, so the
|
|
# ORDER is what is judged. The fake password must never appear in the step's output or any fake's argv.
|
|
r1145_dir="$WORK/r1145"; mkdir -p "$r1145_dir/bin"
|
|
R1145_PW="FAKEPW-r1145-AAAAAAAAAAAAAAAAAAAAAAAAAA"
|
|
cat > "$r1145_dir/bin/openssl" <<EOF
|
|
#!/bin/bash
|
|
echo "$R1145_PW"
|
|
EOF
|
|
cat > "$r1145_dir/bin/curl" <<'EOF'
|
|
#!/bin/bash
|
|
cat >/dev/null # the body (password) arrives on stdin and goes nowhere
|
|
echo "argv: $*" >> "$R1145_DIR/argv"
|
|
case "$*" in
|
|
*recovery-credential*) echo vault >> "$R1145_DIR/calls"; printf '%s' "$FAKE_VAULT" ;;
|
|
*access/ticket*) echo login >> "$R1145_DIR/calls"; printf '%s' "$FAKE_LOGIN" ;;
|
|
esac
|
|
EOF
|
|
cat > "$r1145_dir/bin/chpasswd" <<'EOF'
|
|
#!/bin/bash
|
|
cat >/dev/null
|
|
echo chpasswd >> "$R1145_DIR/calls"
|
|
[[ "$FAKE_CHPASSWD" == ok ]] || exit 1
|
|
echo "root:\$y\$changed:20000::::::" > "$R1145_SHADOW"
|
|
EOF
|
|
printf '#!/bin/bash\nexit 0\n' > "$r1145_dir/bin/sleep"
|
|
chmod +x "$r1145_dir/bin/"*
|
|
sed -n '/^_root_shadow_fingerprint() {/,/^}/p; /^step_break_glass() {/,/^}/p' "$SCRIPT" > "$r1145_dir/step.sh"
|
|
# r1145_run <vault code> <chpasswd ok|fail> <login code> — sets r_rc, r_out, r_calls, r_shadow_changed
|
|
r1145_run() {
|
|
: > "$r1145_dir/calls"; : > "$r1145_dir/argv"; echo 'root:$y$old:20000::::::' > "$r1145_dir/shadow"
|
|
set +e
|
|
r_out=$(PATH="$r1145_dir/bin:$PATH" R1145_DIR="$r1145_dir" R1145_SHADOW="$r1145_dir/shadow" \
|
|
FELHOM_SHADOW_FILE="$r1145_dir/shadow" FAKE_VAULT="$1" FAKE_CHPASSWD="$2" FAKE_LOGIN="$3" bash -c '
|
|
log_step() { echo "STEP $*"; }; log_info() { echo "INFO $*"; }; log_warn() { echo "WARN $*"; }
|
|
log_error() { echo "ERROR $*"; }; log_success() { echo "OK $*"; }; log_skip() { echo "SKIP $*"; }; log_dry() { echo "DRY $*"; }
|
|
die() { echo "DIE $*"; exit 1; }; _state_mark() { echo "MARK $*"; }; _state_has() { return 1; }
|
|
DRY_RUN=false; ROTATE_RECOVERY=true; HOST_ID=h-test; HOST_API_KEY=k-test; HUB_URL=http://127.0.0.1:9
|
|
source "'"$r1145_dir/step.sh"'"; step_break_glass' 2>&1)
|
|
r_rc=$?
|
|
set -e
|
|
r_calls=$(tr '\n' ' ' < "$r1145_dir/calls")
|
|
grep -q 'old' "$r1145_dir/shadow" && r_shadow_changed=no || r_shadow_changed=yes
|
|
}
|
|
r1145_check() { # <name> <want rc 0|1> <want calls> <want shadow changed> <want output substring>
|
|
local ok=true why=""
|
|
[[ "$r_rc" == "$2" || ( "$2" == 1 && "$r_rc" != 0 ) ]] || { ok=false; why+="rc=$r_rc "; }
|
|
[[ "$r_calls" == "$3" ]] || { ok=false; why+="calls='$r_calls' want '$3' "; }
|
|
[[ "$r_shadow_changed" == "$4" ]] || { ok=false; why+="shadow changed=$r_shadow_changed "; }
|
|
[[ "$r_out" == *"$5"* ]] || { ok=false; why+="missing '$5' "; }
|
|
if [[ "$r_out" == *"$R1145_PW"* ]] || grep -qF "$R1145_PW" "$r1145_dir/argv"; then ok=false; why+="the password leaked "; fi
|
|
if grep -qF "k-test" "$r1145_dir/argv" || [[ "$r_out" == *"k-test"* ]]; then ok=false; why+="the host key is in curl's argv or the output "; fi
|
|
if $ok; then verdict PASS "$1"; else verdict FAIL "$1" "$why"; fi
|
|
}
|
|
r1145_run 500 ok 200
|
|
r1145_check "R1145-a vault fails → the password is NOT touched" 1 "vault " no "the root@pam password was NOT changed"
|
|
r1145_run 401 ok 200
|
|
r1145_check "R1145-e a stale host key (vault 401) → the password is NOT touched (review F1)" 1 "vault " no "vault rejected (401)"
|
|
r1145_run 200 ok 200
|
|
r1145_check "R1145-b vault, then set, then the PVE login — in that order" 0 "vault chpasswd login " yes "the PVE login takes it"
|
|
r1145_run 200 fail 200
|
|
r1145_check "R1145-c set fails twice after the vault → the state is said, root unchanged" 1 "vault chpasswd chpasswd " no "STATE: the hub now holds a recovery password that root@pam does NOT have"
|
|
r1145_run 200 ok 401
|
|
r1145_check "R1145-d the PVE login refuses the new value → die, the state is said" 1 "vault chpasswd login " yes "the PVE web login answered HTTP 401"
|
|
|
|
echo ""
|
|
echo "--- F5 appliance guest auto-sizing (VALIDATION-n100) ---"
|
|
# The auto-size log line is emitted right after mode validation (BEFORE any PVE/hub contact), so it
|
|
# is asserted portably on ANY host — the later pipeline dies for lack of PVE, but the sizing already
|
|
# happened. RED-PROOF: pre-fix, appliance passes NO cap → the golden 2 GB default reaches the guest
|
|
# (no auto-size line at all); post-fix, the formula value is logged (and flows into the provision
|
|
# args via cap_args). Host reads are faked via FELHOM_FAKE_MEMTOTAL_MIB / FELHOM_FAKE_NPROC.
|
|
f5_run() { # <memtotal_mib> <nproc> [extra install args...]
|
|
local mt="$1" np="$2"; shift 2
|
|
FELHOM_INSTALL_STATE_DIR="$STATE_OVERRIDE_ENV" FELHOM_FAKE_MEMTOTAL_MIB="$mt" FELHOM_FAKE_NPROC="$np" \
|
|
bash "$SCRIPT" --customer-id t --mode appliance --dry-run "$@" </dev/null 2>&1 || true
|
|
}
|
|
while read -r f5_mt f5_np f5_mem f5_cores; do
|
|
[[ -z "$f5_mt" ]] && continue
|
|
o=$(f5_run "$f5_mt" "$f5_np")
|
|
if echo "$o" | grep -q "auto-sized guest RAM: ${f5_mem} MiB" && echo "$o" | grep -q "auto-sized guest cores: ${f5_cores} "; then
|
|
verdict PASS "F5 auto-size host ${f5_mt}MiB/${f5_np}c -> guest ${f5_mem}MiB/${f5_cores} cores"
|
|
else
|
|
verdict FAIL "F5 auto-size host ${f5_mt}MiB/${f5_np}c -> guest ${f5_mem}MiB/${f5_cores} cores" \
|
|
"got: $(echo "$o" | grep -i auto-sized | tr '\n' '; ')"
|
|
fi
|
|
done <<'F5TABLE'
|
|
8192 8 4096 7
|
|
16384 8 12288 7
|
|
32768 16 28672 15
|
|
F5TABLE
|
|
# Edge: a host below the min clamp — the min (4096) wins but is capped at host-1024.
|
|
o=$(f5_run 4096 2)
|
|
if echo "$o" | grep -q "auto-sized guest RAM: 3072 MiB"; then
|
|
verdict PASS "F5 auto-size small host 4096MiB -> 3072MiB (min capped at host-1024)"
|
|
else
|
|
verdict FAIL "F5 auto-size small host 4096MiB -> 3072MiB" "got: $(echo "$o" | grep -i 'auto-sized guest RAM' | tr '\n' '; ')"
|
|
fi
|
|
# Explicit flags ALWAYS win: NO auto-size line (byte-identical to the pre-fix no-autosize behavior).
|
|
o=$(f5_run 16384 8 --memory 8192 --cores 4)
|
|
if ! echo "$o" | grep -qi "auto-sized"; then
|
|
verdict PASS "F5 explicit --memory/--cores suppress auto-sizing (explicit wins untouched)"
|
|
else
|
|
verdict FAIL "F5 explicit --memory/--cores suppress auto-sizing" "auto-size line present despite explicit flags"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- PVE tier ---"
|
|
if ! command -v pveum >/dev/null 2>&1 || [[ "$(id -u)" != 0 ]]; then
|
|
verdict SKIP "PVE tier (C5 + Scenario A/B dry transcripts + GL8-F6 live)" "needs a PVE host as root — run there"
|
|
else
|
|
# C5: byo with a bogus --acl-storages entry dies in preflight NAMING it (read-only: dies before
|
|
# any hub contact/passphrase and before any mutation; state override active).
|
|
expect_die "C5 byo bogus --acl-storages refused, named" \
|
|
"acl storage(s) not found on this box: definitely-not-a-storage" \
|
|
-- --customer-id t --mode byo --cores 4 --memory 8192 \
|
|
--acl-storages "local definitely-not-a-storage" --dry-run
|
|
|
|
# GL8-F6 live: the byo :53 gate is behaviour-checked against whatever THIS host has on :53
|
|
# (read-only preflight die; state override active). If a resolver is bound (e.g. felhom-pve's
|
|
# leftover dnsmasq), byo preflight MUST refuse with the remediation; if :53 is free, it must pass
|
|
# the gate and die LATER for another reason (never at :53). A PATH-shimmed `ss` proves both arms.
|
|
mkdir -p "$WORK/f6shim"
|
|
printf '#!/bin/sh\nexit 0\n' > "$WORK/f6shim/ss"; chmod 0755 "$WORK/f6shim/ss" # empty :53 output
|
|
if ss -H -ltnup 'sport = :53' 2>/dev/null | grep -q .; then
|
|
expect_die "GL8-F6 live: foreign :53 refused, byo" \
|
|
"a resolver is already bound to :53 on this host — Felhom needs the guest reachable" \
|
|
-- --customer-id t --mode byo --cores 4 --memory 8192 --acl-storages "local" --dry-run
|
|
else
|
|
verdict SKIP "GL8-F6 live: foreign :53 refused, byo" "nothing bound to :53 on this host — cannot exercise the refuse arm"
|
|
fi
|
|
# free-:53 arm (ss shimmed to report nothing): byo preflight passes the :53 gate → dies LATER,
|
|
# never with the :53 message.
|
|
out=$(FELHOM_INSTALL_STATE_DIR="$STATE_OVERRIDE_ENV" PATH="$WORK/f6shim:$PATH" \
|
|
bash "$SCRIPT" --customer-id t --mode byo --cores 4 --memory 8192 --acl-storages "local" --dry-run 2>&1 || true)
|
|
if [[ "$out" != *"already bound to :53"* ]]; then
|
|
verdict PASS "GL8-F6 live: free :53 passes the gate (ss-shim empty)"
|
|
else
|
|
verdict FAIL "GL8-F6 live: free :53 passes the gate (ss-shim empty)" "died at :53 despite an empty ss"
|
|
fi
|
|
|
|
# GL4 H-U: FULL uninstall dry transcript (Scenario A). Read-only: every mutation is dry-printed,
|
|
# the typed confirm takes its dry branch, and the state override keeps the live state.json out.
|
|
# Requires a Felhom guest to target — resolved from felhom_guests-style detection below.
|
|
hu_vmid=$(for id in $( { pct list 2>/dev/null; qm list 2>/dev/null; } | awk "{print \$1}" | grep -E "^[0-9]+$" ); do
|
|
pct config "$id" 2>/dev/null | grep -q "mp=/etc/felhom-bootstrap" && { echo "$id"; break; }
|
|
done)
|
|
if [[ -z "$hu_vmid" ]]; then
|
|
verdict SKIP "GL4 H-U uninstall dry transcript" "no Felhom guest on this host to target"
|
|
else
|
|
run_script --uninstall --vmid "$hu_vmid" --dry-run
|
|
hu_ok=true; hu_why=""
|
|
[[ $rc -eq 0 ]] || { hu_ok=false; hu_why+="rc=$rc "; }
|
|
for want in "felhom-selfupdate-guarded" "kept vs wiped" "NEVER wiped"; do
|
|
[[ "$out" == *"$want"* ]] || { hu_ok=false; hu_why+="missing '$want' "; }
|
|
done
|
|
# MUST NOT: forced/lazy unmounts, or ANY destructive op (mkfs/wipefs/rm) on a drive-data
|
|
# path. Removing the felhom-mkfs-guarded WRAPPER from /usr/local/sbin is legitimate.
|
|
for bad in "umount -l" "umount -f"; do
|
|
[[ "$out" != *"$bad"* ]] || { hu_ok=false; hu_why+="contains '$bad' "; }
|
|
done
|
|
if echo "$out" | grep -E '(mkfs|wipefs|rm |rm -rf).*/mnt/felhom-drives/' >/dev/null; then
|
|
hu_ok=false; hu_why+="destructive op on a /mnt/felhom-drives/ path "
|
|
fi
|
|
# drive umount lines only when child mounts exist — assert conditionally
|
|
if findmnt -rn -o TARGET 2>/dev/null | grep -q '^/mnt/felhom-drives/'; then
|
|
[[ "$out" == *"data stays on the drive"* ]] || { hu_ok=false; hu_why+="missing per-drive umount lines "; }
|
|
fi
|
|
if $hu_ok; then
|
|
verdict PASS "GL4 H-U uninstall dry transcript (guest $hu_vmid; statement + selfupdate removal, no forced ops)"
|
|
else
|
|
verdict FAIL "GL4 H-U uninstall dry transcript" "$hu_why"
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${FELHOM_TEST_CUSTOMER:-}" && -n "${FELHOM_TEST_PASSFILE:-}" && -r "${FELHOM_TEST_PASSFILE:-}" ]]; then
|
|
common=(--customer-id "$FELHOM_TEST_CUSTOMER" --passphrase-file "$FELHOM_TEST_PASSFILE" \
|
|
--vmid 990100 --cores 2 --memory 4096 --dry-run)
|
|
# H-A: appliance dry transcript still walks 4b/8 (regression guard)
|
|
run_script --mode appliance "${common[@]}"
|
|
if [[ $rc -eq 0 && "$out" == *"4b/8"* ]]; then
|
|
verdict PASS "H-A appliance dry transcript contains 4b/8"
|
|
else
|
|
verdict FAIL "H-A appliance dry transcript contains 4b/8" "rc=$rc"
|
|
fi
|
|
# H-B: byo dry transcript — no 4b/8 / chpasswd / recovery-credential; ack + caps + storages present
|
|
run_script --mode byo "${common[@]}"
|
|
hb_ok=true; hb_why=""
|
|
[[ $rc -eq 0 ]] || { hb_ok=false; hb_why+="rc=$rc "; }
|
|
for bad in "4b/8" "chpasswd" "recovery-credential"; do
|
|
[[ "$out" != *"$bad"* ]] || { hb_ok=false; hb_why+="contains '$bad' "; }
|
|
done
|
|
for want in "acknowledge the byo install" "acl storages all present" "-cores 2 -memory 4096" "self-update authority"; do
|
|
[[ "$out" == *"$want"* ]] || { hb_ok=false; hb_why+="missing '$want' "; }
|
|
done
|
|
if $hb_ok; then
|
|
verdict PASS "H-B byo dry transcript (no root@pam path; ack+caps+storages present)"
|
|
else
|
|
verdict FAIL "H-B byo dry transcript" "$hb_why"
|
|
fi
|
|
else
|
|
verdict SKIP "H-A/H-B dry transcripts" "set FELHOM_TEST_CUSTOMER + FELHOM_TEST_PASSFILE (0600 passphrase file) to run"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== $pass passed, $fail failed, $skip skipped ==="
|
|
[[ $fail -eq 0 ]]
|