diff --git a/tests/_run_inside.sh b/tests/_run_inside.sh index 69f51e0..e6c1e97 100644 --- a/tests/_run_inside.sh +++ b/tests/_run_inside.sh @@ -22,6 +22,15 @@ VERBOSE="$1"; shift # complete in seconds, even on a busy phone, so make that failure explicit # rather than making the suite look hung. Override for deliberate long runs. TEST_TIMEOUT_SECONDS="${TEST_TIMEOUT_SECONDS:-120}" +# How many times to start a test whose process never came up at all. Under +# WSL the Godot binary is a Windows executable started across the interop +# boundary, and with six of them going at once that start occasionally just +# fails - exit 127, an empty log, no engine banner, nothing run. It looked +# exactly like a test failure while being a property of the launch, and it +# moved from test to test between runs. A test that genuinely fails always +# gets far enough to print the banner, so the two are easy to tell apart and +# only the launch is worth retrying. +LAUNCH_ATTEMPTS="${LAUNCH_ATTEMPTS:-3}" cd "$PROJECT_DIR" || exit 2 @@ -56,18 +65,37 @@ run_one() { # 3 parallel runs. Pinned, a test simulates the same 60fps whatever else # the phone is doing, and its result no longer depends on how busy the # machine was. - timeout "$TEST_TIMEOUT_SECONDS" "$GODOT_BIN" --headless --fixed-fps 60 \ - -s "res://tests/$name" >"$log" 2>&1 - local rc=$? + local rc=0 + local attempt=1 + while true; do + timeout "$TEST_TIMEOUT_SECONDS" "$GODOT_BIN" --headless --fixed-fps 60 \ + -s "res://tests/$name" >"$log" 2>&1 + rc=$? + # "Never started" is rc != 0 with no banner in the log. Anything that + # ran - passed, failed, crashed mid-way - printed the banner first. + if (( rc == 0 )) || grep -q "Godot Engine" "$log" \ + || (( attempt >= LAUNCH_ATTEMPTS )); then + break + fi + attempt=$(( attempt + 1 )) + sleep 1 + done end=$(date +%s%N) local ms=$(( (end - start) / 1000000 )) + # Retries are reported, never hidden: a test that needed one is telling + # you the machine is struggling, even where it went on to pass. + local retried="" + if (( attempt > 1 )); then + retried=" (started on attempt $attempt)" + fi + # Two independent criteria, because either one alone can lie: a test that # crashes before reaching its verdict exits non-zero without ever printing # FAIL, and a hypothetical `quit(0)` on a failing path would exit clean # while the log says otherwise. if (( rc == 0 )) && grep -q ": PASS" "$log"; then - printf 'PASS %6dms %s\n' "$ms" "$name" > "$outdir/$name.status" + printf 'PASS %6dms %s%s\n' "$ms" "$name" "$retried" > "$outdir/$name.status" elif (( rc == 124 )); then printf 'TIMEOUT %4dms %s (limit %ss)\n' \ "$ms" "$name" "$TEST_TIMEOUT_SECONDS" > "$outdir/$name.status" @@ -78,8 +106,11 @@ run_one() { # indistinguishable from a hang. printf 'KILLED %4dms %s (signal %d - out of memory? try JOBS=1)\n' \ "$ms" "$name" "$(( rc - 128 ))" > "$outdir/$name.status" + elif (( rc != 0 )) && ! grep -q "Godot Engine" "$log"; then + printf 'NOSTART %3dms %s (exit %d after %d attempts - the process never came up)\n' \ + "$ms" "$name" "$rc" "$attempt" > "$outdir/$name.status" else - printf 'FAIL %6dms %s (exit %d)\n' "$ms" "$name" "$rc" > "$outdir/$name.status" + printf 'FAIL %6dms %s (exit %d)%s\n' "$ms" "$name" "$rc" "$retried" > "$outdir/$name.status" fi cat "$outdir/$name.status" } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 3b949ec..6d61a56 100644 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -13,12 +13,12 @@ # Termux the Godot 4.7 Linux arm64 build installed inside the # proot-distro Ubuntu container, which is the only way to run # a desktop Godot on the phone at all. -# WSL the Windows Godot build, run straight from /mnt/c. A Linux -# `godot` on PATH is used first if there is one. +# WSL the Windows Godot build off /mnt/c. A Linux `godot` on PATH +# is used first if there is one. # -# The platform decides which Godot runs and whether the project is copied -# first; the worker count and tests/_run_inside.sh, which does the actual -# running, are shared. +# The platform decides which Godot runs and where the project is copied to. +# Everything else - the copy itself, the worker count, and +# tests/_run_inside.sh, which does the actual running - is shared. # # On Termux, three things make this fast, in descending order of how much # they matter (measured, not guessed - see tests/_profile_startup.gd): @@ -29,9 +29,15 @@ # alone took 8.5s per test, against ~2s from a native path. Nearly every # test loads Main.tscn, and none of that time is gameplay. The copy is # ~1.4s for the whole suite and its .godot import cache is kept between -# runs. WSL skips the copy: a Windows Godot reads C: natively, and the -# only thing copying to ext4 would achieve is putting the project behind -# the \\wsl$ share instead. +# runs. WSL copies too, but to a native NTFS path rather than to ext4: +# a Windows Godot reads C: at full speed and would read \\wsl$ slowly, so +# the copy is for ISOLATION rather than for speed there. Six Godot +# processes churning the project directory is six processes writing +# .godot, creating and deleting files under PlanetSetups/, and doing it +# in a working tree that git, an open editor and the player are also +# using. Under that, single tests failed at random - a delete that did +# not take, a scene that would not instantiate - never the same one +# twice. # 2. Tests run JOBS at a time. Each is its own Godot process - they call # quit() and stamp on the GameState autoload, so they cannot share one - # which makes the suite a pile of independent, single-core-bound jobs on @@ -103,14 +109,31 @@ _pick_jobs() { } JOBS="${JOBS:-$(_pick_jobs)}" VERBOSE="${VERBOSE:-0}" -# The copy only buys anything on Termux (see 1. above), so on WSL the project -# directory IS the run directory whatever IN_PLACE says. -if [[ "$PLATFORM" == "wsl" ]]; then - IN_PLACE=1 -else - IN_PLACE="${IN_PLACE:-0}" -fi -WORK_DIR="${WORK_DIR:-$HOME/.cache/spaceship-tests}" +IN_PLACE="${IN_PLACE:-0}" + +# Where the copy lives. Termux wants its own native filesystem; WSL wants a +# WINDOWS one, because the thing reading it is a Windows binary and $HOME is +# ext4 behind the \\wsl$ share. %LOCALAPPDATA% is the natural home for it; +# if that cannot be read, fall back to a sibling of the project - still on +# the same native drive, still outside the repo. +_default_work_dir() { + if [[ "$PLATFORM" != "wsl" ]]; then + echo "$HOME/.cache/spaceship-tests" + return + fi + local win_local + win_local="$(cmd.exe /c 'echo %LOCALAPPDATA%' 2>/dev/null | tr -d '\r')" + if [[ -n "$win_local" && "$win_local" != *'%'* ]]; then + local unix_local + unix_local="$(wslpath -u "$win_local" 2>/dev/null)" + if [[ -n "$unix_local" ]]; then + echo "$unix_local/spaceship-tests" + return + fi + fi + echo "$(dirname "$PROJECT_DIR")/.spaceship-tests" +} +WORK_DIR="${WORK_DIR:-$(_default_work_dir)}" if [[ -z "${GODOT_BIN:-}" ]]; then if [[ "$PLATFORM" == "wsl" ]]; then