fix: revalidate exiting demo anchors

tools/process-lib.sh

@@ -104,10 +104,12 @@ demo_has_exact_tail() {
 demo_command_matches() {
   local root=$1 pid=$2 kind=$3 expected executable interpreter entry_index
   local -a arguments=()
-  [[ -r "/proc/$pid/cmdline" && -e "/proc/$pid/exe" ]] || return 1
+  # Status 2 means /proc became unreadable while identity was being checked.
-  { mapfile -d '' -t arguments <"/proc/$pid/cmdline"; } 2>/dev/null || return 1
+  # Status 1 remains a definitive command mismatch.
-  ((${#arguments[@]} > 0)) || return 1
+  [[ -r "/proc/$pid/cmdline" && -e "/proc/$pid/exe" ]] || return 2
-  executable=$(readlink -f -- "/proc/$pid/exe" 2>/dev/null) || return 1
+  { mapfile -d '' -t arguments <"/proc/$pid/cmdline"; } 2>/dev/null || return 2
+  ((${#arguments[@]} > 0)) || return 2
+  executable=$(readlink -f -- "/proc/$pid/exe" 2>/dev/null) || return 2
 
   if [[ "$kind" == anvil ]]; then
     expected=$(demo_resolve_command anvil) || return 1
@@ -203,16 +205,20 @@ demo_collect_group_members() {
 }
 
 demo_anchor_allows_signal() {
-  local root=$1 kind=$2 leader=$3 expected_start=$4 expected_pgid=$5 identity_mode=${6:-validated} state start pgid sid
+  local root=$1 kind=$2 leader=$3 expected_start=$4 expected_pgid=$5 identity_mode=${6:-validated}
+  local state start pgid sid command_status attempt
+  local attempts=${DEMO_ANCHOR_RECHECK_ATTEMPTS:-10} interval=${DEMO_ANCHOR_RECHECK_INTERVAL:-0.01}
+  demo_is_uint "$attempts" && ((10#$attempts > 0)) || attempts=10
+  [[ "$interval" =~ ^[0-9]+([.][0-9]+)?$ ]] || interval=0.01
+  for ((attempt = 0; attempt < attempts; attempt++)); do
     if ! demo_read_process_identity "$leader" state start pgid sid; then return 0; fi
     [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_pgid" ]] || return 1
     [[ "$state" == Z ]] && return 0
     [[ "$identity_mode" == raw ]] && return 0
-  demo_process_has_command_line "$leader" || return 0
+    if demo_command_matches "$root" "$leader" "$kind"; then return 0; else command_status=$?; fi
-  if demo_command_matches "$root" "$leader" "$kind"; then return 0; fi
+    ((command_status == 2)) || return 1
-  if ! demo_read_process_identity "$leader" state start pgid sid; then return 0; fi
+    if ((attempt + 1 < attempts)); then sleep "$interval"; fi
-  [[ "$state" == Z ]] && return 0
+  done
-  [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_pgid" ]] || return 1
   return 1
 }
 
@@ -221,10 +227,11 @@ demo_signal_member() {
   if ! demo_read_process_identity "$pid" state start pgid sid; then return 0; fi
   [[ "$state" != Z ]] || return 0
   [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_sid" ]] || return 2
-  if kill -"$signal" -- "$pid"; then return 0; else signal_status=$?; fi
+  if kill -"$signal" -- "$pid" 2>/dev/null; then return 0; else signal_status=$?; fi
   if ! demo_read_process_identity "$pid" state start pgid sid; then return 0; fi
   [[ "$state" != Z ]] || return 0
   [[ "$start" == "$expected_start" && "$pgid" == "$expected_pgid" && "$sid" == "$expected_sid" ]] || return 0
+  printf 'Failed to signal unchanged live PID %s with %s\n' "$pid" "$signal" >&2
   return "$signal_status"
 }
 
@@ -255,13 +262,19 @@ demo_stop_owned_session() {
   for member in "${members[@]}"; do
     IFS=: read -r member_pid member_start <<<"$member"
     [[ "$member_pid" != "$leader" ]] || continue
-    demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode" || return 1
+    if ! demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode"; then
+      printf 'Refusing TERM: %s leader identity changed or remained unverifiable\n' "$kind" >&2
+      return 1
+    fi
     demo_signal_member "$member_pid" "$member_start" "$expected_pgid" "$expected_pgid" TERM || return 1
   done
   for member in "${members[@]}"; do
     IFS=: read -r member_pid member_start <<<"$member"
     [[ "$member_pid" == "$leader" ]] || continue
-    demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode" || return 1
+    if ! demo_anchor_allows_signal "$root" "$kind" "$leader" "$expected_start" "$expected_pgid" "$identity_mode"; then
+      printf 'Refusing TERM: %s leader identity changed or remained unverifiable\n' "$kind" >&2
+      return 1
+    fi
     demo_signal_member "$member_pid" "$member_start" "$expected_pgid" "$expected_pgid" TERM || return 1
   done
   for ((result = 0; result < ${DEMO_TERM_WAIT_ATTEMPTS:-50}; result++)); do

tools/test-process-safety.sh

@@ -8,6 +8,8 @@ TEST_ROOT=$(mktemp -d /tmp/uups-bank-process-test.XXXXXX)
 PATH="$TEST_ROOT/bin:$PATH"
 export DEMO_TERM_WAIT_ATTEMPTS=5
 export DEMO_KILL_WAIT_ATTEMPTS=10
+export DEMO_ANCHOR_RECHECK_ATTEMPTS=10
+export DEMO_ANCHOR_RECHECK_INTERVAL=0.01
 ANVIL_DEV_PHRASE='test test test test test test test test test test test junk'
 ANVIL_EXACT_ARGS=(--host 127.0.0.1 --port 8545 --chain-id 31337 --mnemonic "$ANVIL_DEV_PHRASE")
 declare -a TEST_IDENTITIES=()
@@ -17,7 +19,7 @@ STARTED_PID=
 cleanup() {
   local identity root relative
   for identity in "${TEST_IDENTITIES[@]}"; do test_safe_stop "$identity"; done
-  for root in absent stale nonnumeric wrong-command false-anvil false-vite exact-anvil raw-capture reused atomic matching group mutated partial local-reset base-canonical base-active sentinel; do
+  for root in absent stale nonnumeric wrong-command false-anvil false-vite exact-anvil raw-capture reused atomic matching group term-refusal anchor-race mutated partial local-reset base-canonical base-active sentinel; do
     for relative in \
       .demo/anvil.pid .demo/anvil.start .demo/anvil.pgid .demo/anvil.log \
       .demo/vite.pid .demo/vite.start .demo/vite.pgid .demo/vite.log \
@@ -32,7 +34,9 @@ cleanup() {
       "$TEST_ROOT/$root" 2>/dev/null || true
   done
   rm -f -- "$TEST_ROOT/bin/anvil" "$TEST_ROOT/bin/vite" "$TEST_ROOT/bin/leaderless" \
-    "$TEST_ROOT/vite-child.pid" "$TEST_ROOT/leaderless-child.pid"
+    "$TEST_ROOT/vite-child.pid" "$TEST_ROOT/term-refusal-child.pid" "$TEST_ROOT/anchor-race-child.pid" \
+    "$TEST_ROOT/leaderless-child.pid" "$TEST_ROOT/vanished-signal.err" "$TEST_ROOT/unsignaled-signal.err" \
+    "$TEST_ROOT/term-refusal.err"
   rmdir -- "$TEST_ROOT/bin" "$TEST_ROOT" 2>/dev/null || true
 }
 trap cleanup EXIT
@@ -167,7 +171,7 @@ assert_anvil_args_rejected() {
   demo_stop_raw_launch "$root" anvil "$pid" "$start" "$pgid"
 }
 
-printf '1..21\n'
+printf '1..26\n'
 
 # Catches cleanup treating a missing record as an error or signaling an inferred PID.
 root=$(make_root absent)
@@ -235,6 +239,80 @@ assert_anvil_args_rejected "$root" reordered \
   --port 8545 --host 127.0.0.1 --chain-id 31337 --mnemonic "$ANVIL_DEV_PHRASE"
 pass 'Anvil identity requires the exact complete launch argument tail'
 
+# Catches treating one transient cmdline/exe read failure as a permanent identity mismatch.
+setsid /bin/sleep 120 & recovered_anchor_pid=$!
+track_process "$recovered_anchor_pid"
+recovered_anchor_state='' recovered_anchor_start='' recovered_anchor_pgid='' recovered_anchor_sid=''
+test_process_identity "$recovered_anchor_pid" recovered_anchor_state recovered_anchor_start \
+  recovered_anchor_pgid recovered_anchor_sid
+[[ "$recovered_anchor_state" != Z && "$recovered_anchor_sid" == "$recovered_anchor_pid" ]] \
+  || fail 'recovering anchor fixture was not a live session leader'
+if (
+  anchor_command_attempt=0
+  demo_command_matches() {
+    anchor_command_attempt=$((anchor_command_attempt + 1))
+    ((anchor_command_attempt > 1)) && return 0
+    return 2
+  }
+  demo_anchor_allows_signal "$ROOT" anvil "$recovered_anchor_pid" "$recovered_anchor_start" \
+    "$recovered_anchor_pgid" validated
+); then
+  recovered_anchor_result=0
+else
+  recovered_anchor_result=$?
+fi
+((recovered_anchor_result == 0)) || fail 'a transient anchor command-read failure was not retried'
+assert_alive "$recovered_anchor_pid"
+pass 'a transient anchor command-read failure recovers through exact revalidation'
+
+# Catches failing shutdown when an unverifiable leader vanishes during bounded observation.
+setsid /bin/sleep 120 & vanishing_anchor_pid=$!
+track_process "$vanishing_anchor_pid"
+vanishing_anchor_state='' vanishing_anchor_start='' vanishing_anchor_pgid='' vanishing_anchor_sid=''
+test_process_identity "$vanishing_anchor_pid" vanishing_anchor_state vanishing_anchor_start \
+  vanishing_anchor_pgid vanishing_anchor_sid
+[[ "$vanishing_anchor_state" != Z && "$vanishing_anchor_sid" == "$vanishing_anchor_pid" ]] \
+  || fail 'vanishing anchor fixture was not a live session leader'
+(
+  sleep 0.03
+  builtin kill -TERM -- "$vanishing_anchor_pid" 2>/dev/null || true
+) & vanishing_anchor_killer=$!
+if (
+  demo_command_matches() { return 2; }
+  demo_anchor_allows_signal "$ROOT" anvil "$vanishing_anchor_pid" "$vanishing_anchor_start" \
+    "$vanishing_anchor_pgid" validated
+); then
+  vanishing_anchor_result=0
+else
+  vanishing_anchor_result=$?
+fi
+wait "$vanishing_anchor_killer"
+wait "$vanishing_anchor_pid" 2>/dev/null || true
+((vanishing_anchor_result == 0)) || fail 'an exiting unverifiable anchor made shutdown fatal'
+pass 'an unverifiable anchor is accepted only after it vanishes or becomes a zombie'
+
+# Catches accepting a persistent live anchor merely because cmdline/exe cannot be read.
+setsid /bin/sleep 120 & unreadable_anchor_pid=$!
+track_process "$unreadable_anchor_pid"
+unreadable_anchor_state='' unreadable_anchor_start='' unreadable_anchor_pgid='' unreadable_anchor_sid=''
+test_process_identity "$unreadable_anchor_pid" unreadable_anchor_state unreadable_anchor_start \
+  unreadable_anchor_pgid unreadable_anchor_sid
+[[ "$unreadable_anchor_state" != Z && "$unreadable_anchor_sid" == "$unreadable_anchor_pid" ]] \
+  || fail 'unreadable anchor fixture was not a live session leader'
+if (
+  demo_process_has_command_line() { return 1; }
+  demo_command_matches() { return 2; }
+  demo_anchor_allows_signal "$ROOT" anvil "$unreadable_anchor_pid" "$unreadable_anchor_start" \
+    "$unreadable_anchor_pgid" validated
+); then
+  unreadable_anchor_result=0
+else
+  unreadable_anchor_result=$?
+fi
+((unreadable_anchor_result != 0)) || fail 'a stable live anchor with unreadable command identity was accepted'
+assert_alive "$unreadable_anchor_pid"
+pass 'a stable live anchor remains refused when command identity is unverifiable'
+
 # Catches treating an ESRCH-like signal race as fatal after the captured member has exited.
 setsid /bin/bash -c 'trap "exit 0" TERM; while :; do sleep 0.01; done' & vanished_pid=$!
 track_process "$vanished_pid"
@@ -251,7 +329,8 @@ kill() {
   done
   return 1
 }
-if demo_signal_member "$vanished_pid" "$vanished_start" "$vanished_pgid" "$vanished_sid" TERM; then
+if demo_signal_member "$vanished_pid" "$vanished_start" "$vanished_pgid" "$vanished_sid" TERM \
+  2>"$TEST_ROOT/vanished-signal.err"; then
   vanished_result=0
 else
   vanished_result=$?
@@ -259,6 +338,7 @@ fi
 unset -f kill
 wait "$vanished_pid" 2>/dev/null || true
 ((vanished_result == 0)) || fail 'a vanished member made its raced signal fatal'
+[[ ! -s "$TEST_ROOT/vanished-signal.err" ]] || fail 'a benign vanished-member signal race emitted raw stderr'
 pass 'a failed signal is idempotent after the captured member vanishes'
 
 # Catches swallowing a real signal failure while the exact captured member remains live.
@@ -268,13 +348,17 @@ unsignaled_state='' unsignaled_start='' unsignaled_pgid='' unsignaled_sid=''
 test_process_identity "$unsignaled_pid" unsignaled_state unsignaled_start unsignaled_pgid unsignaled_sid
 [[ "$unsignaled_state" != Z ]] || fail 'unchanged signal-failure fixture exited early'
 kill() { return 1; }
-if demo_signal_member "$unsignaled_pid" "$unsignaled_start" "$unsignaled_pgid" "$unsignaled_sid" TERM; then
+if demo_signal_member "$unsignaled_pid" "$unsignaled_start" "$unsignaled_pgid" "$unsignaled_sid" TERM \
+  2>"$TEST_ROOT/unsignaled-signal.err"; then
   unsignaled_result=0
 else
   unsignaled_result=$?
 fi
 unset -f kill
 ((unsignaled_result != 0)) || fail 'a failed signal to the unchanged live member was accepted'
+[[ $(<"$TEST_ROOT/unsignaled-signal.err") == \
+  "Failed to signal unchanged live PID $unsignaled_pid with TERM" ]] \
+  || fail 'an unchanged-live signal failure lacked its controlled diagnostic'
 assert_alive "$unsignaled_pid"
 pass 'a failed signal remains fatal for the same live member tuple'
 
@@ -348,6 +432,80 @@ assert_dead "$child_pid"
 assert_alive "$unrelated_pid"
 pass 'bounded KILL removes a TERM-resistant owned child while an unrelated group survives'
 
+# Catches a silent TERM-loop refusal after exact command identity definitively changes.
+root=$(make_root term-refusal)
+term_refusal_helper="$TEST_ROOT/bin/anvil"
+term_refusal_child_file="$TEST_ROOT/term-refusal-child.pid"
+cat >"$term_refusal_helper" <<'HELPER'
+#!/usr/bin/env bash
+sleep 120 &
+printf '%s\n' "$!" >"$DEMO_CHILD_PID_FILE"
+wait
+sleep 120
+HELPER
+chmod +x "$term_refusal_helper"
+DEMO_CHILD_PID_FILE="$term_refusal_child_file" setsid "$term_refusal_helper" "${ANVIL_EXACT_ARGS[@]}" \
+  & term_refusal_pid=$!
+track_process "$term_refusal_pid"
+for _ in {1..50}; do [[ -s "$term_refusal_child_file" ]] && break; sleep 0.02; done
+[[ -s "$term_refusal_child_file" ]] || fail 'TERM-refusal helper child did not start'
+term_refusal_start=$(start_tick "$term_refusal_pid")
+if (
+  anchor_command_attempt=0
+  demo_command_matches() {
+    anchor_command_attempt=$((anchor_command_attempt + 1))
+    ((anchor_command_attempt <= 2)) && return 0
+    return 1
+  }
+  demo_stop_launch "$root" anvil "$term_refusal_pid" "$term_refusal_start" "$term_refusal_pid"
+) 2>"$TEST_ROOT/term-refusal.err"; then
+  term_refusal_result=0
+else
+  term_refusal_result=$?
+fi
+((term_refusal_result != 0)) || fail 'a definitive TERM-loop anchor mismatch was accepted'
+[[ $(<"$TEST_ROOT/term-refusal.err") == \
+  "Refusing TERM: anvil leader identity changed or remained unverifiable" ]] \
+  || fail 'a TERM-loop anchor refusal lacked its controlled diagnostic'
+assert_alive "$term_refusal_pid"
+pass 'a definitive TERM-loop anchor mismatch is refused with a controlled diagnostic'
+
+# Catches the child-exit/leader-exit race becoming fatal during transient command-read loss.
+root=$(make_root anchor-race)
+anchor_race_helper="$TEST_ROOT/bin/anvil"
+anchor_race_child_file="$TEST_ROOT/anchor-race-child.pid"
+cat >"$anchor_race_helper" <<'HELPER'
+#!/usr/bin/env bash
+sleep 120 &
+printf '%s\n' "$!" >"$DEMO_CHILD_PID_FILE"
+wait
+sleep 0.05
+HELPER
+chmod +x "$anchor_race_helper"
+DEMO_CHILD_PID_FILE="$anchor_race_child_file" setsid "$anchor_race_helper" "${ANVIL_EXACT_ARGS[@]}" \
+  & anchor_race_pid=$!
+track_process "$anchor_race_pid"
+for _ in {1..50}; do [[ -s "$anchor_race_child_file" ]] && break; sleep 0.02; done
+[[ -s "$anchor_race_child_file" ]] || fail 'anchor-race helper child did not start'
+anchor_race_start=$(start_tick "$anchor_race_pid")
+if (
+  anchor_command_attempt=0
+  demo_command_matches() {
+    anchor_command_attempt=$((anchor_command_attempt + 1))
+    ((anchor_command_attempt <= 2)) && return 0
+    return 2
+  }
+  demo_stop_launch "$root" anvil "$anchor_race_pid" "$anchor_race_start" "$anchor_race_pid"
+); then
+  anchor_race_result=0
+else
+  anchor_race_result=$?
+fi
+wait "$anchor_race_pid" 2>/dev/null || true
+((anchor_race_result == 0)) || fail 'child exit made the concurrently exiting leader fatal'
+assert_dead "$anchor_race_pid"
+pass 'child-exit cleanup tolerates transient identity loss while the leader exits'
+
 # Catches escalating after the recorded leader changes to a different command identity.
 root=$(make_root mutated)
 mutating_helper="$TEST_ROOT/bin/anvil"