diff --git a/tests/_run_inside.sh b/tests/_run_inside.sh index e6c1e97..a38e4a5 100644 --- a/tests/_run_inside.sh +++ b/tests/_run_inside.sh @@ -22,16 +22,35 @@ VERBOSE="$1"; shift # complete in seconds, even on a busy phone, so make that failure explicit # rather than making the suite look hung. Override for deliberate long runs. TEST_TIMEOUT_SECONDS="${TEST_TIMEOUT_SECONDS:-120}" -# How many times to start a test whose process never came up at all. Under -# WSL the Godot binary is a Windows executable started across the interop -# boundary, and with six of them going at once that start occasionally just -# fails - exit 127, an empty log, no engine banner, nothing run. It looked -# exactly like a test failure while being a property of the launch, and it -# moved from test to test between runs. A test that genuinely fails always -# gets far enough to print the banner, so the two are easy to tell apart and -# only the launch is worth retrying. +# How many times to start a test whose process never got as far as saying +# anything. Under WSL the Godot binary is a Windows executable started across +# the interop boundary, and with six of them going at once that start +# occasionally just fails. It has been seen two ways: exit 127 with a +# completely empty log, and a non-zero exit after the engine banner and not +# one line more. Both moved from test to test between runs and both passed +# alone, so both are properties of the launch rather than of the test. +# +# The line that separates them from a real failure is OUTPUT, not the exit +# code and not the banner: every test says something before it stops - its +# own FAIL verdict, a SCRIPT ERROR, an engine complaint. A log holding +# nothing but the banner is a run that did not happen. This can only ever +# mask a fault that is both silent AND intermittent; a deterministic crash +# still fails all LAUNCH_ATTEMPTS and is reported. LAUNCH_ATTEMPTS="${LAUNCH_ATTEMPTS:-3}" + +# Did the process produce any line of its own? The engine banner and blank +# lines are what a bare launch prints by itself and do not count. +# +# Tests the OUTPUT rather than grep's exit status on purpose: `grep -q -v` +# does not agree across greps - ugrep, which is what `grep` resolves to on +# at least one machine this suite runs on, reports success for an empty file +# and failure for a file full of matching lines. A non-empty result is the +# same in every implementation. +_said_anything() { + [[ -n "$(grep -v -e '^[[:space:]]*$' -e '^Godot Engine' "$1")" ]] +} + cd "$PROJECT_DIR" || exit 2 tests=("$@") @@ -71,9 +90,9 @@ run_one() { timeout "$TEST_TIMEOUT_SECONDS" "$GODOT_BIN" --headless --fixed-fps 60 \ -s "res://tests/$name" >"$log" 2>&1 rc=$? - # "Never started" is rc != 0 with no banner in the log. Anything that - # ran - passed, failed, crashed mid-way - printed the banner first. - if (( rc == 0 )) || grep -q "Godot Engine" "$log" \ + # "Never started" is rc != 0 with a log that says nothing of its own. + # Anything that ran - passed, failed, crashed mid-way - said something. + if (( rc == 0 )) || _said_anything "$log" \ || (( attempt >= LAUNCH_ATTEMPTS )); then break fi @@ -106,7 +125,7 @@ run_one() { # indistinguishable from a hang. printf 'KILLED %4dms %s (signal %d - out of memory? try JOBS=1)\n' \ "$ms" "$name" "$(( rc - 128 ))" > "$outdir/$name.status" - elif (( rc != 0 )) && ! grep -q "Godot Engine" "$log"; then + elif (( rc != 0 )) && ! _said_anything "$log"; then printf 'NOSTART %3dms %s (exit %d after %d attempts - the process never came up)\n' \ "$ms" "$name" "$rc" "$attempt" > "$outdir/$name.status" else