spaceship/tests/_run_inside.sh

#!/bin/bash
# The half of the runner that actually runs Godot - do not call this
# directly, call tests/run_tests.sh instead.
#
# On Termux it runs INSIDE the proot-distro Ubuntu container. It exists as
# its own file so the whole suite runs in ONE `proot-distro login`: entering
# the container costs ~0.5s, which used to be paid 38 times over. On WSL
# there is no container to enter and run_tests.sh calls it directly - the
# scheduling, timeouts and verdicts below are the same either way, which is
# why the Godot binary arrives as an argument rather than being hardcoded.
#
# Args: <godot_bin> <project_dir> <jobs> <verbose> <test_file_name>...

set -uo pipefail

GODOT_BIN="$1"; shift
PROJECT_DIR="$1"; shift
JOBS="$1"; shift
VERBOSE="$1"; shift
# A broken test can otherwise leave SceneTree alive after an initialization
# error, printing the same error every frame forever. Gameplay tests normally
# complete in seconds, even on a busy phone, so make that failure explicit
# rather than making the suite look hung. Override for deliberate long runs.
TEST_TIMEOUT_SECONDS="${TEST_TIMEOUT_SECONDS:-120}"
# How many times to start a test whose process never got as far as saying
# anything. Under WSL the Godot binary is a Windows executable started across
# the interop boundary, and with six of them going at once that start
# occasionally just fails. It has been seen two ways: exit 127 with a
# completely empty log, and a non-zero exit after the engine banner and not
# one line more. Both moved from test to test between runs and both passed
# alone, so both are properties of the launch rather than of the test.
#
# The line that separates them from a real failure is OUTPUT, not the exit
# code and not the banner: every test says something before it stops - its
# own FAIL verdict, a SCRIPT ERROR, an engine complaint. A log holding
# nothing but the banner is a run that did not happen. This can only ever
# mask a fault that is both silent AND intermittent; a deterministic crash
# still fails all LAUNCH_ATTEMPTS and is reported.
LAUNCH_ATTEMPTS="${LAUNCH_ATTEMPTS:-3}"


# Did the process produce any line of its own? The engine banner and blank
# lines are what a bare launch prints by itself and do not count.
#
# Tests the OUTPUT rather than grep's exit status on purpose: `grep -q -v`
# does not agree across greps - ugrep, which is what `grep` resolves to on
# at least one machine this suite runs on, reports success for an empty file
# and failure for a file full of matching lines. A non-empty result is the
# same in every implementation.
_said_anything() {
  [[ -n "$(grep -v -e '^[[:space:]]*$' -e '^Godot Engine' "$1")" ]]
}

cd "$PROJECT_DIR" || exit 2

tests=("$@")
total=${#tests[@]}
if (( total == 0 )); then
  echo "no tests matched"
  exit 2
fi

outdir="$(mktemp -d)"
trap 'rm -rf "$outdir"' EXIT
REPORT_DIR="${TEST_REPORT_DIR:-$PROJECT_DIR/.test-results}"
mkdir -p "$REPORT_DIR"

# Import once, serially, before anything runs in parallel. Godot writes to
# .godot/ while importing; several instances discovering a cold cache at the
# same moment would race on it. After this the workers only read it.
"$GODOT_BIN" --headless --import --quit >/dev/null 2>&1

run_one() {
  local name="$1"
  local log="$outdir/$name.log"
  local start end
  start=$(date +%s%N)
  # --fixed-fps pins delta to exactly 1/60 instead of measuring the wall
  # clock. Without it a test's `elapsed` counts REAL seconds, so running
  # several at once stretches every frame and the gameplay under test sees
  # deltas it never sees in the game: ships overshoot their braking curve,
  # settle late, and tests time out for no reason but load. That made
  # test_camera_settles and test_planet_prompt_map_interaction fail on 2 of
  # 3 parallel runs. Pinned, a test simulates the same 60fps whatever else
  # the phone is doing, and its result no longer depends on how busy the
  # machine was.
  local rc=0
  local attempt=1
  while true; do
    timeout "$TEST_TIMEOUT_SECONDS" "$GODOT_BIN" --headless --fixed-fps 60 \
      -s "res://tests/$name" >"$log" 2>&1
    rc=$?
    # "Never started" is rc != 0 with a log that says nothing of its own.
    # Anything that ran - passed, failed, crashed mid-way - said something.
    if (( rc == 0 )) || _said_anything "$log" \
        || (( attempt >= LAUNCH_ATTEMPTS )); then
      break
    fi
    attempt=$(( attempt + 1 ))
    sleep 1
  done
  end=$(date +%s%N)
  local ms=$(( (end - start) / 1000000 ))

  # Retries are reported, never hidden: a test that needed one is telling
  # you the machine is struggling, even where it went on to pass.
  local retried=""
  if (( attempt > 1 )); then
    retried=" (started on attempt $attempt)"
  fi

  # Two independent criteria, because either one alone can lie: a test that
  # crashes before reaching its verdict exits non-zero without ever printing
  # FAIL, and a hypothetical `quit(0)` on a failing path would exit clean
  # while the log says otherwise.
  if (( rc == 0 )) && grep -q ": PASS" "$log"; then
    printf 'PASS %6dms  %s%s\n' "$ms" "$name" "$retried" > "$outdir/$name.status"
  elif (( rc == 124 )); then
    printf 'TIMEOUT %4dms  %s (limit %ss)\n' \
      "$ms" "$name" "$TEST_TIMEOUT_SECONDS" > "$outdir/$name.status"
  elif (( rc == 137 || rc == 139 )); then
    # Killed, not failed - almost always the kernel reclaiming memory. Worth
    # saying plainly: a killed worker and a failing one both simply produced
    # no verdict before, so a whole suite dying to the OOM killer was
    # indistinguishable from a hang.
    printf 'KILLED %4dms  %s (signal %d - out of memory? try JOBS=1)\n' \
      "$ms" "$name" "$(( rc - 128 ))" > "$outdir/$name.status"
  elif (( rc != 0 )) && ! _said_anything "$log"; then
    printf 'NOSTART %3dms  %s (exit %d after %d attempts - the process never came up)\n' \
      "$ms" "$name" "$rc" "$attempt" > "$outdir/$name.status"
  else
    printf 'FAIL %6dms  %s (exit %d)%s\n' "$ms" "$name" "$rc" "$retried" > "$outdir/$name.status"
  fi
  cat "$outdir/$name.status"
}

running=0
for name in "${tests[@]}"; do
  run_one "$name" &
  running=$(( running + 1 ))
  if (( running >= JOBS )); then
    wait -n
    running=$(( running - 1 ))
  fi
done
wait

failed=0
for name in "${tests[@]}"; do
  if ! grep -q '^PASS' "$outdir/$name.status" 2>/dev/null; then
    failed=$(( failed + 1 ))
    echo
    echo "=== $name FAILED, full output ==="
    cat "$outdir/$name.log" 2>/dev/null
  elif [[ "$VERBOSE" == "1" ]]; then
    echo
    echo "=== $name ==="
    cat "$outdir/$name.log" 2>/dev/null
  fi
done

echo
echo "$(( total - failed ))/$total passed"
# Keep the concise verdicts after the temporary logs are removed. This is
# especially useful from Termux, where a long-running child can outlive the
# terminal command that launched it and its final buffered output is lost.
rm -f "$REPORT_DIR"/*.status
cp "$outdir"/*.status "$REPORT_DIR"/
(( failed == 0 ))