spaceship/tests/_run_inside.sh

#!/bin/bash
# The half of the runner that actually runs Godot - do not call this
# directly, call tests/run_tests.sh instead.
#
# On Termux it runs INSIDE the proot-distro Ubuntu container. It exists as
# its own file so the whole suite runs in ONE `proot-distro login`: entering
# the container costs ~0.5s, which used to be paid 38 times over. On WSL
# there is no container to enter and run_tests.sh calls it directly - the
# scheduling, timeouts and verdicts below are the same either way, which is
# why the Godot binary arrives as an argument rather than being hardcoded.
#
# Args: <godot_bin> <project_dir> <jobs> <verbose> <test_file_name>...

set -uo pipefail

GODOT_BIN="$1"; shift
PROJECT_DIR="$1"; shift
JOBS="$1"; shift
VERBOSE="$1"; shift
# A broken test can otherwise leave SceneTree alive after an initialization
# error, printing the same error every frame forever. Gameplay tests normally
# complete in seconds, even on a busy phone, so make that failure explicit
# rather than making the suite look hung. Override for deliberate long runs.
TEST_TIMEOUT_SECONDS="${TEST_TIMEOUT_SECONDS:-120}"
# How many times to start a test whose process never came up at all. Under
# WSL the Godot binary is a Windows executable started across the interop
# boundary, and with six of them going at once that start occasionally just
# fails - exit 127, an empty log, no engine banner, nothing run. It looked
# exactly like a test failure while being a property of the launch, and it
# moved from test to test between runs. A test that genuinely fails always
# gets far enough to print the banner, so the two are easy to tell apart and
# only the launch is worth retrying.
LAUNCH_ATTEMPTS="${LAUNCH_ATTEMPTS:-3}"

cd "$PROJECT_DIR" || exit 2

tests=("$@")
total=${#tests[@]}
if (( total == 0 )); then
  echo "no tests matched"
  exit 2
fi

outdir="$(mktemp -d)"
trap 'rm -rf "$outdir"' EXIT
REPORT_DIR="${TEST_REPORT_DIR:-$PROJECT_DIR/.test-results}"
mkdir -p "$REPORT_DIR"

# Import once, serially, before anything runs in parallel. Godot writes to
# .godot/ while importing; several instances discovering a cold cache at the
# same moment would race on it. After this the workers only read it.
"$GODOT_BIN" --headless --import --quit >/dev/null 2>&1

run_one() {
  local name="$1"
  local log="$outdir/$name.log"
  local start end
  start=$(date +%s%N)
  # --fixed-fps pins delta to exactly 1/60 instead of measuring the wall
  # clock. Without it a test's `elapsed` counts REAL seconds, so running
  # several at once stretches every frame and the gameplay under test sees
  # deltas it never sees in the game: ships overshoot their braking curve,
  # settle late, and tests time out for no reason but load. That made
  # test_camera_settles and test_planet_prompt_map_interaction fail on 2 of
  # 3 parallel runs. Pinned, a test simulates the same 60fps whatever else
  # the phone is doing, and its result no longer depends on how busy the
  # machine was.
  local rc=0
  local attempt=1
  while true; do
    timeout "$TEST_TIMEOUT_SECONDS" "$GODOT_BIN" --headless --fixed-fps 60 \
      -s "res://tests/$name" >"$log" 2>&1
    rc=$?
    # "Never started" is rc != 0 with no banner in the log. Anything that
    # ran - passed, failed, crashed mid-way - printed the banner first.
    if (( rc == 0 )) || grep -q "Godot Engine" "$log" \
        || (( attempt >= LAUNCH_ATTEMPTS )); then
      break
    fi
    attempt=$(( attempt + 1 ))
    sleep 1
  done
  end=$(date +%s%N)
  local ms=$(( (end - start) / 1000000 ))

  # Retries are reported, never hidden: a test that needed one is telling
  # you the machine is struggling, even where it went on to pass.
  local retried=""
  if (( attempt > 1 )); then
    retried=" (started on attempt $attempt)"
  fi

  # Two independent criteria, because either one alone can lie: a test that
  # crashes before reaching its verdict exits non-zero without ever printing
  # FAIL, and a hypothetical `quit(0)` on a failing path would exit clean
  # while the log says otherwise.
  if (( rc == 0 )) && grep -q ": PASS" "$log"; then
    printf 'PASS %6dms  %s%s\n' "$ms" "$name" "$retried" > "$outdir/$name.status"
  elif (( rc == 124 )); then
    printf 'TIMEOUT %4dms  %s (limit %ss)\n' \
      "$ms" "$name" "$TEST_TIMEOUT_SECONDS" > "$outdir/$name.status"
  elif (( rc == 137 || rc == 139 )); then
    # Killed, not failed - almost always the kernel reclaiming memory. Worth
    # saying plainly: a killed worker and a failing one both simply produced
    # no verdict before, so a whole suite dying to the OOM killer was
    # indistinguishable from a hang.
    printf 'KILLED %4dms  %s (signal %d - out of memory? try JOBS=1)\n' \
      "$ms" "$name" "$(( rc - 128 ))" > "$outdir/$name.status"
  elif (( rc != 0 )) && ! grep -q "Godot Engine" "$log"; then
    printf 'NOSTART %3dms  %s (exit %d after %d attempts - the process never came up)\n' \
      "$ms" "$name" "$rc" "$attempt" > "$outdir/$name.status"
  else
    printf 'FAIL %6dms  %s (exit %d)%s\n' "$ms" "$name" "$rc" "$retried" > "$outdir/$name.status"
  fi
  cat "$outdir/$name.status"
}

running=0
for name in "${tests[@]}"; do
  run_one "$name" &
  running=$(( running + 1 ))
  if (( running >= JOBS )); then
    wait -n
    running=$(( running - 1 ))
  fi
done
wait

failed=0
for name in "${tests[@]}"; do
  if ! grep -q '^PASS' "$outdir/$name.status" 2>/dev/null; then
    failed=$(( failed + 1 ))
    echo
    echo "=== $name FAILED, full output ==="
    cat "$outdir/$name.log" 2>/dev/null
  elif [[ "$VERBOSE" == "1" ]]; then
    echo
    echo "=== $name ==="
    cat "$outdir/$name.log" 2>/dev/null
  fi
done

echo
echo "$(( total - failed ))/$total passed"
# Keep the concise verdicts after the temporary logs are removed. This is
# especially useful from Termux, where a long-running child can outlive the
# terminal command that launched it and its final buffered output is lost.
rm -f "$REPORT_DIR"/*.status
cp "$outdir"/*.status "$REPORT_DIR"/
(( failed == 0 ))