From eb2fd5245a73b74532f2cec5b018261f4e1cc3b3 Mon Sep 17 00:00:00 2001 From: golem Date: Fri, 21 Aug 2026 05:12:50 -0600 Subject: [PATCH] fix: revalidate exiting demo anchors --- tools/process-lib.sh | 47 ++++++---- tools/test-process-safety.sh | 168 +++++++++++++++++++++++++++++++++-- 2 files changed, 193 insertions(+), 22 deletions(-) diff --git a/tools/process-lib.sh b/tools/process-lib.sh index 4267922..c41b199 100755 --- a/tools/process-lib.sh +++ b/tools/process-lib.sh @@ -104,10 +104,12 @@ demo_has_exact_tail() { demo_command_matches() { local root=$1 pid=$2 kind=$3 expected executable interpreter entry_index local -a arguments=() - [[ -r "/proc/$pid/cmdline" && -e "/proc/$pid/exe" ]] || return 1 - { mapfile -d '' -t arguments <"/proc/$pid/cmdline"; } 2>/dev/null || return 1 - ((${#arguments[@]} > 0)) || return 1 - executable=$(readlink -f -- "/proc/$pid/exe" 2>/dev/null) || return 1 + # Status 2 means /proc became unreadable while identity was being checked. + # Status 1 remains a definitive command mismatch. + [[ -r "/proc/$pid/cmdline" && -e "/proc/$pid/exe" ]] || return 2 + { mapfile -d '' -t arguments <"/proc/$pid/cmdline"; } 2>/dev/null || return 2 + ((${#arguments[@]} > 0)) || return 2 + executable=$(readlink -f -- "/proc/$pid/exe" 2>/dev/null) || return 2 if [[ "$kind" == anvil ]]; then expected=$(demo_resolve_command anvil) || return 1 @@ -203,16 +205,20 @@ demo_collect_group_members() { } demo_anchor_allows_signal() { - local root=$1 kind=$2 leader=$3 expected_start=$4 expected_pgid=$5 identity_mode=${6:-validated} state start pgid sid - if ! demo_read_process_identity "$leader" state start pgid sid; then return 0; fi - [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_pgid" ]] || return 1 - [[ "$state" == Z ]] && return 0 - [[ "$identity_mode" == raw ]] && return 0 - demo_process_has_command_line "$leader" || return 0 - if demo_command_matches "$root" "$leader" "$kind"; then return 0; fi - if ! demo_read_process_identity "$leader" state start pgid sid; then return 0; fi - [[ "$state" == Z ]] && return 0 - [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_pgid" ]] || return 1 + local root=$1 kind=$2 leader=$3 expected_start=$4 expected_pgid=$5 identity_mode=${6:-validated} + local state start pgid sid command_status attempt + local attempts=${DEMO_ANCHOR_RECHECK_ATTEMPTS:-10} interval=${DEMO_ANCHOR_RECHECK_INTERVAL:-0.01} + demo_is_uint "$attempts" && ((10#$attempts > 0)) || attempts=10 + [[ "$interval" =~ ^[0-9]+([.][0-9]+)?$ ]] || interval=0.01 + for ((attempt = 0; attempt < attempts; attempt++)); do + if ! demo_read_process_identity "$leader" state start pgid sid; then return 0; fi + [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_pgid" ]] || return 1 + [[ "$state" == Z ]] && return 0 + [[ "$identity_mode" == raw ]] && return 0 + if demo_command_matches "$root" "$leader" "$kind"; then return 0; else command_status=$?; fi + ((command_status == 2)) || return 1 + if ((attempt + 1 < attempts)); then sleep "$interval"; fi + done return 1 } @@ -221,10 +227,11 @@ demo_signal_member() { if ! demo_read_process_identity "$pid" state start pgid sid; then return 0; fi [[ "$state" != Z ]] || return 0 [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_sid" ]] || return 2 - if kill -"$signal" -- "$pid"; then return 0; else signal_status=$?; fi + if kill -"$signal" -- "$pid" 2>/dev/null; then return 0; else signal_status=$?; fi if ! demo_read_process_identity "$pid" state start pgid sid; then return 0; fi [[ "$state" != Z ]] || return 0 [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_sid" ]] || return 0 + printf 'Failed to signal unchanged live PID %s with %s\n' "$pid" "$signal" >&2 return "$signal_status" } @@ -255,13 +262,19 @@ demo_stop_owned_session() { for member in "${members[@]}"; do IFS=: read -r member_pid member_start <<<"$member" [[ "$member_pid" != "$leader" ]] || continue - demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode" || return 1 + if ! demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode"; then + printf 'Refusing TERM: %s leader identity changed or remained unverifiable\n' "$kind" >&2 + return 1 + fi demo_signal_member "$member_pid" "$member_start" "$expected_pgid" "$expected_pgid" TERM || return 1 done for member in "${members[@]}"; do IFS=: read -r member_pid member_start <<<"$member" [[ "$member_pid" == "$leader" ]] || continue - demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode" || return 1 + if ! demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode"; then + printf 'Refusing TERM: %s leader identity changed or remained unverifiable\n' "$kind" >&2 + return 1 + fi demo_signal_member "$member_pid" "$member_start" "$expected_pgid" "$expected_pgid" TERM || return 1 done for ((result = 0; result < ${DEMO_TERM_WAIT_ATTEMPTS:-50}; result++)); do diff --git a/tools/test-process-safety.sh b/tools/test-process-safety.sh index 09e4b0b..8cc9d9d 100755 --- a/tools/test-process-safety.sh +++ b/tools/test-process-safety.sh @@ -8,6 +8,8 @@ TEST_ROOT=$(mktemp -d /tmp/uups-bank-process-test.XXXXXX) PATH="$TEST_ROOT/bin:$PATH" export DEMO_TERM_WAIT_ATTEMPTS=5 export DEMO_KILL_WAIT_ATTEMPTS=10 +export DEMO_ANCHOR_RECHECK_ATTEMPTS=10 +export DEMO_ANCHOR_RECHECK_INTERVAL=0.01 ANVIL_DEV_PHRASE='test test test test test test test test test test test junk' ANVIL_EXACT_ARGS=(--host 127.0.0.1 --port 8545 --chain-id 31337 --mnemonic "$ANVIL_DEV_PHRASE") declare -a TEST_IDENTITIES=() @@ -17,7 +19,7 @@ STARTED_PID= cleanup() { local identity root relative for identity in "${TEST_IDENTITIES[@]}"; do test_safe_stop "$identity"; done - for root in absent stale nonnumeric wrong-command false-anvil false-vite exact-anvil raw-capture reused atomic matching group mutated partial local-reset base-canonical base-active sentinel; do + for root in absent stale nonnumeric wrong-command false-anvil false-vite exact-anvil raw-capture reused atomic matching group term-refusal anchor-race mutated partial local-reset base-canonical base-active sentinel; do for relative in \ .demo/anvil.pid .demo/anvil.start .demo/anvil.pgid .demo/anvil.log \ .demo/vite.pid .demo/vite.start .demo/vite.pgid .demo/vite.log \ @@ -32,7 +34,9 @@ cleanup() { "$TEST_ROOT/$root" 2>/dev/null || true done rm -f -- "$TEST_ROOT/bin/anvil" "$TEST_ROOT/bin/vite" "$TEST_ROOT/bin/leaderless" \ - "$TEST_ROOT/vite-child.pid" "$TEST_ROOT/leaderless-child.pid" + "$TEST_ROOT/vite-child.pid" "$TEST_ROOT/term-refusal-child.pid" "$TEST_ROOT/anchor-race-child.pid" \ + "$TEST_ROOT/leaderless-child.pid" "$TEST_ROOT/vanished-signal.err" "$TEST_ROOT/unsignaled-signal.err" \ + "$TEST_ROOT/term-refusal.err" rmdir -- "$TEST_ROOT/bin" "$TEST_ROOT" 2>/dev/null || true } trap cleanup EXIT @@ -167,7 +171,7 @@ assert_anvil_args_rejected() { demo_stop_raw_launch "$root" anvil "$pid" "$start" "$pgid" } -printf '1..21\n' +printf '1..26\n' # Catches cleanup treating a missing record as an error or signaling an inferred PID. root=$(make_root absent) @@ -235,6 +239,80 @@ assert_anvil_args_rejected "$root" reordered \ --port 8545 --host 127.0.0.1 --chain-id 31337 --mnemonic "$ANVIL_DEV_PHRASE" pass 'Anvil identity requires the exact complete launch argument tail' +# Catches treating one transient cmdline/exe read failure as a permanent identity mismatch. +setsid /bin/sleep 120 & recovered_anchor_pid=$! +track_process "$recovered_anchor_pid" +recovered_anchor_state='' recovered_anchor_start='' recovered_anchor_pgid='' recovered_anchor_sid='' +test_process_identity "$recovered_anchor_pid" recovered_anchor_state recovered_anchor_start \ + recovered_anchor_pgid recovered_anchor_sid +[[ "$recovered_anchor_state" != Z && "$recovered_anchor_sid" == "$recovered_anchor_pid" ]] \ + || fail 'recovering anchor fixture was not a live session leader' +if ( + anchor_command_attempt=0 + demo_command_matches() { + anchor_command_attempt=$((anchor_command_attempt + 1)) + ((anchor_command_attempt > 1)) && return 0 + return 2 + } + demo_anchor_allows_signal "$ROOT" anvil "$recovered_anchor_pid" "$recovered_anchor_start" \ + "$recovered_anchor_pgid" validated +); then + recovered_anchor_result=0 +else + recovered_anchor_result=$? +fi +((recovered_anchor_result == 0)) || fail 'a transient anchor command-read failure was not retried' +assert_alive "$recovered_anchor_pid" +pass 'a transient anchor command-read failure recovers through exact revalidation' + +# Catches failing shutdown when an unverifiable leader vanishes during bounded observation. +setsid /bin/sleep 120 & vanishing_anchor_pid=$! +track_process "$vanishing_anchor_pid" +vanishing_anchor_state='' vanishing_anchor_start='' vanishing_anchor_pgid='' vanishing_anchor_sid='' +test_process_identity "$vanishing_anchor_pid" vanishing_anchor_state vanishing_anchor_start \ + vanishing_anchor_pgid vanishing_anchor_sid +[[ "$vanishing_anchor_state" != Z && "$vanishing_anchor_sid" == "$vanishing_anchor_pid" ]] \ + || fail 'vanishing anchor fixture was not a live session leader' +( + sleep 0.03 + builtin kill -TERM -- "$vanishing_anchor_pid" 2>/dev/null || true +) & vanishing_anchor_killer=$! +if ( + demo_command_matches() { return 2; } + demo_anchor_allows_signal "$ROOT" anvil "$vanishing_anchor_pid" "$vanishing_anchor_start" \ + "$vanishing_anchor_pgid" validated +); then + vanishing_anchor_result=0 +else + vanishing_anchor_result=$? +fi +wait "$vanishing_anchor_killer" +wait "$vanishing_anchor_pid" 2>/dev/null || true +((vanishing_anchor_result == 0)) || fail 'an exiting unverifiable anchor made shutdown fatal' +pass 'an unverifiable anchor is accepted only after it vanishes or becomes a zombie' + +# Catches accepting a persistent live anchor merely because cmdline/exe cannot be read. +setsid /bin/sleep 120 & unreadable_anchor_pid=$! +track_process "$unreadable_anchor_pid" +unreadable_anchor_state='' unreadable_anchor_start='' unreadable_anchor_pgid='' unreadable_anchor_sid='' +test_process_identity "$unreadable_anchor_pid" unreadable_anchor_state unreadable_anchor_start \ + unreadable_anchor_pgid unreadable_anchor_sid +[[ "$unreadable_anchor_state" != Z && "$unreadable_anchor_sid" == "$unreadable_anchor_pid" ]] \ + || fail 'unreadable anchor fixture was not a live session leader' +if ( + demo_process_has_command_line() { return 1; } + demo_command_matches() { return 2; } + demo_anchor_allows_signal "$ROOT" anvil "$unreadable_anchor_pid" "$unreadable_anchor_start" \ + "$unreadable_anchor_pgid" validated +); then + unreadable_anchor_result=0 +else + unreadable_anchor_result=$? +fi +((unreadable_anchor_result != 0)) || fail 'a stable live anchor with unreadable command identity was accepted' +assert_alive "$unreadable_anchor_pid" +pass 'a stable live anchor remains refused when command identity is unverifiable' + # Catches treating an ESRCH-like signal race as fatal after the captured member has exited. setsid /bin/bash -c 'trap "exit 0" TERM; while :; do sleep 0.01; done' & vanished_pid=$! track_process "$vanished_pid" @@ -251,7 +329,8 @@ kill() { done return 1 } -if demo_signal_member "$vanished_pid" "$vanished_start" "$vanished_pgid" "$vanished_sid" TERM; then +if demo_signal_member "$vanished_pid" "$vanished_start" "$vanished_pgid" "$vanished_sid" TERM \ + 2>"$TEST_ROOT/vanished-signal.err"; then vanished_result=0 else vanished_result=$? @@ -259,6 +338,7 @@ fi unset -f kill wait "$vanished_pid" 2>/dev/null || true ((vanished_result == 0)) || fail 'a vanished member made its raced signal fatal' +[[ ! -s "$TEST_ROOT/vanished-signal.err" ]] || fail 'a benign vanished-member signal race emitted raw stderr' pass 'a failed signal is idempotent after the captured member vanishes' # Catches swallowing a real signal failure while the exact captured member remains live. @@ -268,13 +348,17 @@ unsignaled_state='' unsignaled_start='' unsignaled_pgid='' unsignaled_sid='' test_process_identity "$unsignaled_pid" unsignaled_state unsignaled_start unsignaled_pgid unsignaled_sid [[ "$unsignaled_state" != Z ]] || fail 'unchanged signal-failure fixture exited early' kill() { return 1; } -if demo_signal_member "$unsignaled_pid" "$unsignaled_start" "$unsignaled_pgid" "$unsignaled_sid" TERM; then +if demo_signal_member "$unsignaled_pid" "$unsignaled_start" "$unsignaled_pgid" "$unsignaled_sid" TERM \ + 2>"$TEST_ROOT/unsignaled-signal.err"; then unsignaled_result=0 else unsignaled_result=$? fi unset -f kill ((unsignaled_result != 0)) || fail 'a failed signal to the unchanged live member was accepted' +[[ $(<"$TEST_ROOT/unsignaled-signal.err") == \ + "Failed to signal unchanged live PID $unsignaled_pid with TERM" ]] \ + || fail 'an unchanged-live signal failure lacked its controlled diagnostic' assert_alive "$unsignaled_pid" pass 'a failed signal remains fatal for the same live member tuple' @@ -348,6 +432,80 @@ assert_dead "$child_pid" assert_alive "$unrelated_pid" pass 'bounded KILL removes a TERM-resistant owned child while an unrelated group survives' +# Catches a silent TERM-loop refusal after exact command identity definitively changes. +root=$(make_root term-refusal) +term_refusal_helper="$TEST_ROOT/bin/anvil" +term_refusal_child_file="$TEST_ROOT/term-refusal-child.pid" +cat >"$term_refusal_helper" <<'HELPER' +#!/usr/bin/env bash +sleep 120 & +printf '%s\n' "$!" >"$DEMO_CHILD_PID_FILE" +wait +sleep 120 +HELPER +chmod +x "$term_refusal_helper" +DEMO_CHILD_PID_FILE="$term_refusal_child_file" setsid "$term_refusal_helper" "${ANVIL_EXACT_ARGS[@]}" \ + & term_refusal_pid=$! +track_process "$term_refusal_pid" +for _ in {1..50}; do [[ -s "$term_refusal_child_file" ]] && break; sleep 0.02; done +[[ -s "$term_refusal_child_file" ]] || fail 'TERM-refusal helper child did not start' +term_refusal_start=$(start_tick "$term_refusal_pid") +if ( + anchor_command_attempt=0 + demo_command_matches() { + anchor_command_attempt=$((anchor_command_attempt + 1)) + ((anchor_command_attempt <= 2)) && return 0 + return 1 + } + demo_stop_launch "$root" anvil "$term_refusal_pid" "$term_refusal_start" "$term_refusal_pid" +) 2>"$TEST_ROOT/term-refusal.err"; then + term_refusal_result=0 +else + term_refusal_result=$? +fi +((term_refusal_result != 0)) || fail 'a definitive TERM-loop anchor mismatch was accepted' +[[ $(<"$TEST_ROOT/term-refusal.err") == \ + "Refusing TERM: anvil leader identity changed or remained unverifiable" ]] \ + || fail 'a TERM-loop anchor refusal lacked its controlled diagnostic' +assert_alive "$term_refusal_pid" +pass 'a definitive TERM-loop anchor mismatch is refused with a controlled diagnostic' + +# Catches the child-exit/leader-exit race becoming fatal during transient command-read loss. +root=$(make_root anchor-race) +anchor_race_helper="$TEST_ROOT/bin/anvil" +anchor_race_child_file="$TEST_ROOT/anchor-race-child.pid" +cat >"$anchor_race_helper" <<'HELPER' +#!/usr/bin/env bash +sleep 120 & +printf '%s\n' "$!" >"$DEMO_CHILD_PID_FILE" +wait +sleep 0.05 +HELPER +chmod +x "$anchor_race_helper" +DEMO_CHILD_PID_FILE="$anchor_race_child_file" setsid "$anchor_race_helper" "${ANVIL_EXACT_ARGS[@]}" \ + & anchor_race_pid=$! +track_process "$anchor_race_pid" +for _ in {1..50}; do [[ -s "$anchor_race_child_file" ]] && break; sleep 0.02; done +[[ -s "$anchor_race_child_file" ]] || fail 'anchor-race helper child did not start' +anchor_race_start=$(start_tick "$anchor_race_pid") +if ( + anchor_command_attempt=0 + demo_command_matches() { + anchor_command_attempt=$((anchor_command_attempt + 1)) + ((anchor_command_attempt <= 2)) && return 0 + return 2 + } + demo_stop_launch "$root" anvil "$anchor_race_pid" "$anchor_race_start" "$anchor_race_pid" +); then + anchor_race_result=0 +else + anchor_race_result=$? +fi +wait "$anchor_race_pid" 2>/dev/null || true +((anchor_race_result == 0)) || fail 'child exit made the concurrently exiting leader fatal' +assert_dead "$anchor_race_pid" +pass 'child-exit cleanup tolerates transient identity loss while the leader exits' + # Catches escalating after the recorded leader changes to a different command identity. root=$(make_root mutated) mutating_helper="$TEST_ROOT/bin/anvil"