chore: migrate more tests to new test suite (#12809)

This PR migrates most remaining tests to the new test suite. It also
completes the migration of directories like `tests/lean/run`, meaning
that PRs trying to add tests to those old directories will now fail.

.gitattributes

@@ -5,9 +5,3 @@ stage0/** binary linguist-generated
 # The following file is often manually edited, so do show it in diffs
 stage0/src/stdlib_flags.h -binary -linguist-generated
 doc/std/grove/GroveStdlib/Generated/** linguist-generated
-# These files should not have line endings translated on Windows, because
-# it throws off parser tests. Later lines override earlier ones, so the
-# runner code is still treated as ordinary text.
-tests/lean/docparse/* eol=lf
-tests/lean/docparse/*.lean eol=auto
-tests/lean/docparse/*.sh eol=auto

doc/.gitignore

@@ -1 +0,0 @@
-out

doc/examples/.gitignore

@@ -0,0 +1,2 @@
+*.out.produced
+*.exit.produced

doc/examples/bintree.lean.out.expected

@@ -0,0 +1,2 @@
+Tree.node (Tree.node (Tree.leaf) 1 "one" (Tree.leaf)) 2 "two" (Tree.node (Tree.leaf) 3 "three" (Tree.leaf))
+[(1, "one"), (2, "two"), (3, "three")]

doc/examples/compiler/run_test

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+source ../../../tests/env_test.sh
+source "$TEST_DIR/util.sh"
+
+leanmake --always-make bin
+
+exec_capture test.lean \
+  ./build/bin/test hello world
+
+check_exit test.lean
+check_out test.lean

doc/examples/compiler/test.lean.out.expected

@@ -0,0 +1 @@
+[hello, world]

doc/examples/interp.lean.out.expected

@@ -0,0 +1,3 @@
+30
+interp.lean:146:4: warning: declaration uses `sorry`
+3628800

doc/examples/palindromes.lean.out.expected

@@ -0,0 +1,2 @@
+true
+false

doc/examples/phoas.lean.out.expected

@@ -0,0 +1,2 @@
+"(((fun x_1 => (fun x_2 => (x_1 + x_2))) 1) 2)"
+"((((fun x_1 => (fun x_2 => (x_1 + x_2))) 1) 2) + 5)"

doc/examples/run_test

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+source ../../tests/env_test.sh
+source "$TEST_DIR/util.sh"
+
+exec_capture "$1" \
+  lean -Dlinter.all=false "$1"
+
+check_exit "$1"
+check_out "$1"

doc/examples/test_single.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-source ../../tests/common.sh
-
-exec_check_raw lean -Dlinter.all=false "$f"

src/Lean/Server/Test/Runner.lean

@@ -265,7 +265,7 @@ def ident : Parser Name := do
   return xs.foldl .str $ .mkSimple head
 
 def patchUri (s : String) : IO String := do
-  let patterns := #["/src/Init/", "/src/Lean/", "/src/Std/", "/tests/lean/interactive/"]
+  let patterns := #["/src/Init/", "/src/Lean/", "/src/Std/", "/tests/misc_dir/"]
   let some path := System.Uri.fileUriToPath? s
     | return s
   let path ← try

tests/CMakeLists.txt

@@ -50,9 +50,9 @@ endblock()
 ## Helper functions ##
 ######################
 
-function(check_test_bench_scripts DIR DIR_ABS)
+function(check_test_bench_scripts DIR)
-  set(RUN_TEST "${DIR_ABS}/run_test")
+  set(RUN_TEST "${DIR}/run_test")
-  set(RUN_BENCH "${DIR_ABS}/run_bench")
+  set(RUN_BENCH "${DIR}/run_bench")
 
   set(RUN_TEST_EXISTS FALSE)
   set(RUN_BENCH_EXISTS FALSE)
@@ -64,6 +64,7 @@ function(check_test_bench_scripts DIR DIR_ABS)
   endif()
 
   if(NOT RUN_TEST_EXISTS AND NOT RUN_BENCH_EXISTS)
+    cmake_path(RELATIVE_PATH DIR)
     message(FATAL_ERROR "${DIR}: Found neither a run_test nor a run_bench file")
     return()
   endif()
@@ -77,10 +78,12 @@ endfunction()
 
 function(check_bench_argument DIR ARGS_BENCH RUN_BENCH_EXISTS)
   if(RUN_BENCH_EXISTS AND NOT ARGS_BENCH)
+    cmake_path(RELATIVE_PATH DIR)
     message(FATAL_ERROR "${DIR}: run_bench file found, BENCH argument must be specified")
     return()
   endif()
   if(NOT RUN_BENCH_EXISTS AND ARGS_BENCH)
+    cmake_path(RELATIVE_PATH DIR)
     message(FATAL_ERROR "${DIR}: BENCH argument specified but no run_bench file found")
     return()
   endif()
@@ -115,9 +118,9 @@ endfunction()
 # added to the list specified by the BENCH argument.
 function(add_test_pile DIR GLOB)
   cmake_parse_arguments(ARGS "" BENCH "" ${ARGN})
-  set(DIR_ABS "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}")
+  cmake_path(ABSOLUTE_PATH DIR NORMALIZE)
 
-  check_test_bench_scripts("${DIR}" "${DIR_ABS}")
+  check_test_bench_scripts("${DIR}")
   check_bench_argument("${DIR}" "${ARGS_BENCH}" "${RUN_BENCH_EXISTS}")
 
   # The test files' individual measurement files that will later be combined
@@ -125,38 +128,40 @@ function(add_test_pile DIR GLOB)
   set(MEASUREMENTS_FILES "")
 
   # Iterate over all files matching the glob
-  file(GLOB TEST_FILES "${DIR_ABS}/${GLOB}")
+  file(GLOB TEST_FILES "${DIR}/${GLOB}")
-  foreach(FILE_ABS IN LISTS TEST_FILES)
+  foreach(FILE IN LISTS TEST_FILES)
-    # Path relative to source directory
+    cmake_path(RELATIVE_PATH FILE OUTPUT_VARIABLE FILE_IN_SOURCE_DIR)
-    cmake_path(RELATIVE_PATH FILE_ABS BASE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE FILE)
+    cmake_path(RELATIVE_PATH FILE BASE_DIRECTORY "${DIR}" OUTPUT_VARIABLE FILE_IN_DIR)
-    # Path relative to pile directory
-    cmake_path(RELATIVE_PATH FILE_ABS BASE_DIRECTORY "${DIR_ABS}" OUTPUT_VARIABLE FILE_NAME)
 
-    if(RUN_TEST_EXISTS AND NOT EXISTS "${FILE_ABS}.no_test")
+    if(FILE_IN_DIR MATCHES "^run_(test|bench)")
+      continue()
+    endif()
+
+    if(RUN_TEST_EXISTS AND NOT EXISTS "${FILE}.no_test")
       add_test(
-        NAME "${FILE}"
+        NAME "${FILE_IN_SOURCE_DIR}"
-        WORKING_DIRECTORY "${DIR_ABS}"
+        WORKING_DIRECTORY "${DIR}"
         # On Windows, we can't call the file directly, hence we always use bash.
-        COMMAND bash "${RUN_TEST}" "${FILE_NAME}"
+        COMMAND bash "${RUN_TEST}" "${FILE_IN_DIR}"
       )
     endif()
 
-    if(RUN_BENCH_EXISTS AND NOT EXISTS "${FILE_ABS}.no_bench")
+    if(RUN_BENCH_EXISTS AND NOT EXISTS "${FILE}.no_bench")
-      set(MEASUREMENTS_FILE "${FILE_ABS}.measurements.jsonl")
+      set(MEASUREMENTS_FILE "${FILE}.measurements.jsonl")
       list(APPEND MEASUREMENTS_FILES "${MEASUREMENTS_FILE}")
       add_custom_command(
         OUTPUT "${MEASUREMENTS_FILE}"
-        WORKING_DIRECTORY "${DIR_ABS}"
+        WORKING_DIRECTORY "${DIR}"
         COMMAND "${CMAKE_COMMAND}" -E remove -f "${MEASUREMENTS_FILE}"
         # On Windows, we can't call the file directly, hence we always use bash.
-        COMMAND bash "${RUN_BENCH}" "${FILE_NAME}"
+        COMMAND bash "${RUN_BENCH}" "${FILE_IN_DIR}"
       )
     endif()
   endforeach()
 
   # Combine measurements
   if(RUN_BENCH_EXISTS)
-    set(MEASUREMENTS_FILE "${DIR_ABS}/measurements.jsonl")
+    set(MEASUREMENTS_FILE "${DIR}/measurements.jsonl")
     list(APPEND "${ARGS_BENCH}" "${MEASUREMENTS_FILE}")
     set("${ARGS_BENCH}" "${${ARGS_BENCH}}" PARENT_SCOPE)
     add_combined_measurements("${MEASUREMENTS_FILE}" "${MEASUREMENTS_FILES}")
@@ -167,16 +172,17 @@ endfunction()
 # alongside any additional files or subdirectories required by that test.
 function(add_test_dir DIR)
   cmake_parse_arguments(ARGS "" BENCH "" ${ARGN})
-  set(DIR_ABS "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}")
+  cmake_path(ABSOLUTE_PATH DIR NORMALIZE)
 
-  check_test_bench_scripts("${DIR}" "${DIR_ABS}")
+  check_test_bench_scripts("${DIR}")
   check_bench_argument("${DIR}" "${ARGS_BENCH}" "${RUN_BENCH_EXISTS}")
 
   # Add as test
   if(RUN_TEST_EXISTS)
+    cmake_path(RELATIVE_PATH DIR OUTPUT_VARIABLE DIR_IN_SOURCE_DIR)
     add_test(
-      NAME "${DIR}"
+      NAME "${DIR_IN_SOURCE_DIR}"
-      WORKING_DIRECTORY "${DIR_ABS}"
+      WORKING_DIRECTORY "${DIR}"
       # On Windows, we can't call the file directly, hence we always use bash.
       COMMAND bash "${RUN_TEST}"
     )
@@ -184,12 +190,12 @@ function(add_test_dir DIR)
 
   # Add as benchmark
   if(RUN_BENCH_EXISTS)
-    set(MEASUREMENTS_FILE "${DIR_ABS}/measurements.jsonl")
+    set(MEASUREMENTS_FILE "${DIR}/measurements.jsonl")
     list(APPEND "${ARGS_BENCH}" "${MEASUREMENTS_FILE}")
     set("${ARGS_BENCH}" "${${ARGS_BENCH}}" PARENT_SCOPE)
     add_custom_command(
       OUTPUT "${MEASUREMENTS_FILE}"
-      WORKING_DIRECTORY "${DIR_ABS}"
+      WORKING_DIRECTORY "${DIR}"
       COMMAND "${CMAKE_COMMAND}" -E remove -f "${MEASUREMENTS_FILE}"
       # On Windows, we can't call the file directly, hence we always use bash.
       COMMAND bash "${RUN_BENCH}"
@@ -197,6 +203,16 @@ function(add_test_dir DIR)
   endif()
 endfunction()
 
+function(add_dir_of_test_dirs DIR)
+  cmake_path(ABSOLUTE_PATH DIR NORMALIZE)
+  file(GLOB TEST_DIRS "${DIR}/*")
+  foreach(TEST_DIR IN LISTS TEST_DIRS)
+    if(IS_DIRECTORY "${TEST_DIR}")
+      add_test_dir("${TEST_DIR}")
+    endif()
+  endforeach()
+endfunction()
+
 # Benchmarks are split into two parts which should be roughly equal in total runtime.
 # In radar, each part is run on a different runner.
 set(PART1 "")
@@ -206,110 +222,6 @@ set(PART2 "")
 ## Tests and benchmarks ##
 ##########################
 
-# LEAN TESTS
-file(GLOB LEANTESTS "${LEAN_SOURCE_DIR}/../tests/lean/*.lean")
-foreach(T ${LEANTESTS})
-  if(NOT T MATCHES "\\.#")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN RUN TESTS
-file(GLOB LEANRUNTESTS "${LEAN_SOURCE_DIR}/../tests/lean/run/*.lean")
-foreach(T ${LEANRUNTESTS})
-  if(NOT T MATCHES "\\.#")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean/run"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN RUN doc/examples
-file(GLOB LEANDOCEXS "${LEAN_SOURCE_DIR}/../doc/examples/*.lean")
-foreach(T ${LEANDOCEXS})
-  if(NOT T MATCHES "\\.#")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../doc/examples"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN COMPILER TESTS
-file(GLOB LEANCOMPTESTS "${LEAN_SOURCE_DIR}/../tests/compiler/*.lean")
-foreach(T ${LEANCOMPTESTS})
-  get_filename_component(T_NAME ${T} NAME)
-  cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-  add_test(
-    NAME "${T_PATH}"
-    WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/compiler"
-    COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-  )
-endforeach(T)
-
-add_test(
-  NAME leancomptest_foreign
-  WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/compiler/foreign"
-  COMMAND bash -c "${LEAN_BIN}/leanmake --always-make"
-)
-add_test(
-  NAME leancomptest_doc_example
-  WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../doc/examples/compiler"
-  COMMAND bash -c "export ${TEST_VARS}; leanmake --always-make bin && ./build/bin/test hello world"
-)
-
-# LEAN INTERPRETER TESTS
-file(GLOB LEANINTERPTESTS "${LEAN_SOURCE_DIR}/../tests/compiler/*.lean")
-foreach(T ${LEANINTERPTESTS})
-  if(NOT EXISTS "${T}.no_interpreter")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH} (interpreted)"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/compiler"
-      COMMAND bash -c "${TEST_VARS} ./test_single_interpret.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN BENCHMARK TESTS
-# do not test all .lean files in bench/
-file(GLOB LEANBENCHTESTS "${LEAN_SOURCE_DIR}/../tests/bench/*.lean.expected.out")
-foreach(T_OUT ${LEANBENCHTESTS})
-  string(REPLACE ".expected.out" "" T ${T_OUT})
-  get_filename_component(T_NAME ${T} NAME)
-  cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-  add_test(
-    NAME "${T_PATH}"
-    WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/bench"
-    COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-  )
-endforeach(T_OUT)
-
-file(GLOB LEANINTERPTESTS "${LEAN_SOURCE_DIR}/../tests/plugin/*.lean")
-foreach(T ${LEANINTERPTESTS})
-  get_filename_component(T_NAME ${T} NAME)
-  cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-  add_test(
-    NAME "${T_PATH}"
-    WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/plugin"
-    COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-  )
-endforeach(T)
-
 # LEAN PACKAGE TESTS
 file(GLOB LEANPKGTESTS "${LEAN_SOURCE_DIR}/../tests/pkg/*")
 foreach(T ${LEANPKGTESTS})
@@ -320,62 +232,6 @@ foreach(T ${LEANPKGTESTS})
   endif()
 endforeach(T)
 
-# LEAN SERVER TESTS
-file(GLOB LEANTESTS "${LEAN_SOURCE_DIR}/../tests/lean/server/*.lean")
-foreach(T ${LEANTESTS})
-  if(NOT T MATCHES "\\.#")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean/server"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN INTERACTIVE SERVER TESTS
-file(GLOB LEANTESTS "${LEAN_SOURCE_DIR}/../tests/lean/interactive/*.lean")
-foreach(T ${LEANTESTS})
-  if(NOT T MATCHES "\\.#" AND NOT T MATCHES "run.lean")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean/interactive"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN INTERACTIVE PROJECT SERVER TESTS
-file(GLOB_RECURSE LEANTESTS "${LEAN_SOURCE_DIR}/../tests/lean/interactive/projects/*Test.lean")
-foreach(T ${LEANTESTS})
-  if(NOT T MATCHES "\\.#")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean/interactive/projects"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T}"
-    )
-  endif()
-endforeach(T)
-
-# LEAN DOCSTRING PARSER TESTS
-file(GLOB_RECURSE LEANTESTS "${LEAN_SOURCE_DIR}/../tests/lean/docparse/*_[0-9][0-9][0-9][0-9]")
-foreach(T ${LEANTESTS})
-  if(NOT T MATCHES "\\.#" AND NOT T MATCHES "run.lean")
-    get_filename_component(T_NAME ${T} NAME)
-    cmake_path(RELATIVE_PATH T BASE_DIRECTORY ${LEAN_SOURCE_DIR}/.. OUTPUT_VARIABLE T_PATH)
-    add_test(
-      NAME "${T_PATH}"
-      WORKING_DIRECTORY "${LEAN_SOURCE_DIR}/../tests/lean/docparse"
-      COMMAND bash -c "${TEST_VARS} ./test_single.sh ${T_NAME}"
-    )
-  endif()
-endforeach(T)
-
 # Create a lake test for each test and examples subdirectory of `lake`
 # which contains a `test.sh` file, excluding the following test(s):
 # bootstrap: too slow
@@ -410,18 +266,25 @@ endforeach(T)
 add_test(NAME lint.py COMMAND python3 lint.py WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
 set_tests_properties(lint.py PROPERTIES RUN_SERIAL TRUE)
 
+add_test_pile(../doc/examples *.lean)
 add_test_pile(compile *.lean BENCH PART2)
 add_test_pile(compile_bench *.lean BENCH PART2)
+add_test_pile(docparse *.txt)
 add_test_pile(elab *.lean)
 add_test_pile(elab_bench *.lean BENCH PART2)
 add_test_pile(elab_fail *.lean)
 add_test_pile(misc *.sh)
 add_test_pile(misc_bench *.sh BENCH PART2)
+add_test_pile(server *.lean)
+add_test_pile(server_interactive *.lean)
 
+add_test_dir(../doc/examples/compiler)
 add_test_dir(bench/build BENCH PART1)
 add_test_dir(bench/size BENCH PART1)
 add_test_dir(lake_bench/inundation BENCH PART2)
 
+add_dir_of_test_dirs(misc_dir)
+
 #######################
 ## Benchmark targets ##
 #######################

tests/README.md

@@ -33,12 +33,18 @@ Benchmarks belonging to the old framework are not included in this description.
 - `compile_bench`:
   Benchmarks that compile lean files and measure the execution of the resulting binary,
   as well as optionally run the same lean file through the interpreter.
+  These are also executed as part of the test suite, and `.out.expected` files are ignored when benchmarking.
+- `docparse`:
+  Docstring parsing tests.
 - `elab`:
   Tests that elaborate lean files without executing them, verifying the resulting output.
 - `elab_fail`:
   Like `elab`, but expecting an exit code of 1 instead of 0.
 - `elab_bench`:
   Like `elab`, but measuring the elaboration performance.
+  These are also executed as part of the test suite, and `.out.expected` files are ignored when benchmarking.
+- `server`, `server_interactive`:
+  Test LSP server requests.
 - `lake_bench`:
   Benchmark directories that measure lake performance.
 - `misc`:
@@ -94,7 +100,7 @@ Run an individual benchmark by `cd`-ing into its directory and then using
 ./run_bench testfile # in a test pile
 ```
 
-## How to write a test or benchmark?
+## How to write a test?
 
 If your test fits one of the existing test piles:
 
@@ -119,6 +125,28 @@ Otherwise, create a new test directory or pile:
    by updating the directory structure section above.
 5. Optionally update [`lint.py`](lint.py) if it makes sense.
 
+## How to write a benchmark?
+
+When writing a benchmark, consider that most benchmarks are also executed as tests.
+You can check that this is the case if a `run_test` script exists next to the `run_bench` script in the directory.
+When run as benchmark, the problem instance should be large enough to result in reliable measurements.
+When run as test, the problem instance should be as small as possible, but large enough to still test the different code paths.
+
+The main mechanism to scale problem instances is the `TEST_BENCH` environment variable.
+It is unset in tests and set to `1` in benchmarks.
+Inside your benchmark, check whether the variable exists and adjust the problem size accordingly.
+
+Inside the `compile_bench` directory, there is a second mechanism:
+Using a `.init.sh` file to pass command line arguments to your test.
+This is useful if you also want to generate graphs for your parametric benchmarks.
+See [`tests/compile_bench/binarytrees.lean`](tests/compile_bench/binarytrees.lean) as an example.
+
+If you want custom metrics aside from the usual `instructions`, `wall-clock`, ...
+inside the `elab_bench` or `compile_bench` directories,
+you can print them to stdout in the format `measurement: <name> <value>[ <unit>]`,
+e.g. `measurement: size 1337 B` or `measurement: iterations 42`.
+See [`tests/compile_bench/ilean_roundtrip.lean`](tests/compile_bench/ilean_roundtrip.lean) as an example.
+
 ## How to fix existing tests after your change breaks them?
 
 If the tests break because the expected output differs from the actual output,
@@ -251,3 +279,35 @@ These bash variables (set via `<file>.init.sh`) are used by the run script:
 - `TEST_ARGS`:
   A bash array of arguments to the compiled (or interpreted) program.
   Check `TEST_BENCH` if you want to specify more intense parameters for benchmarks.
+
+## The `interactive` test pile
+
+These tests are designed to test LSP server requests at a given position in the input file.
+Each `.lean` file contains comments that indicate how to simulate a client request at a position,
+using a `--^` point to the line position.
+
+Example:
+
+```lean,ignore
+open Foo in
+theorem tst2 (h : a ≤ b) : a + 2 ≤ b + 2 :=
+Bla.
+  --^ completion
+```
+
+In this example, the test driver will simulate an auto-completion request at `Bla.`.
+The expected output is stored in the corresponding `.out.expected` file
+in the json format that is part of the
+[Language Server Protocol](https://microsoft.github.io/language-server-protocol/).
+
+This can also be used to test the following additional requests:
+
+```
+--^ textDocument/hover
+--^ textDocument/typeDefinition
+--^ textDocument/definition
+--^ $/lean/plainGoal
+--^ $/lean/plainTermGoal
+--^ insert: ...
+--^ collectDiagnostics
+```

tests/compiler/foreign/.gitignore

@@ -1 +0,0 @@
-build

tests/compiler/foreign/Main.lean

@@ -1,11 +0,0 @@
-import Main.S
-
-def main : IO Unit := do
-  IO.println (mkS 10 20 "hello").addXY
-  IO.println (mkS 10 20 "hello").string
-  appendToGlobalS "foo"
-  appendToGlobalS "bla"
-  getGlobalString >>= IO.println
-  updateGlobalS (mkS 0 0 "world")
-  getGlobalString >>= IO.println
-  pure ()

tests/compiler/foreign/Main/S.lean

@@ -1,11 +0,0 @@
-opaque SPointed : NonemptyType
-def S : Type := SPointed.type
-instance : Nonempty S := SPointed.property
-
-@[extern "lean_mk_S"] opaque mkS (x y : UInt32) (s : @& String) : S
-@[extern "lean_S_add_x_y"] opaque S.addXY (s : @& S) : UInt32
-@[extern "lean_S_string"] opaque S.string (s : @& S) : String
--- The following 3 externs have side effects. Thus, we put them in IO.
-@[extern "lean_S_global_append"] opaque appendToGlobalS (str : @& String) : IO Unit
-@[extern "lean_S_global_string"] opaque getGlobalString : IO String
-@[extern "lean_S_update_global"] opaque updateGlobalS (s : @& S) : IO Unit

tests/compiler/foreign/Makefile

@@ -1,40 +0,0 @@
-CXX ?= c++
-CPPFLAGS = -O3
-# TODO: use Lake instead
-include lean.mk
-
-CPP_SRCS = myfuns.cpp
-CPP_OBJS = $(addprefix $(OUT)/testcpp/,$(CPP_SRCS:.cpp=.o))
-
-all: run_test run_interp
-
-$(OUT)/testcpp/%.o: %.cpp
-	@mkdir -p "$(@D)"
-	$(CXX) -std=c++14 -c -o $@ $< $(CPPFLAGS) `leanc --print-cflags`
-
-# to avoid conflicts between the system C++ stdlib needed by the above object file and the internal one used in the Lean runtime,
-# we need to dynamically link the Lean runtime.
-
-ifeq ($(OS),Windows_NT)
-# make S.so find testcpp.so
-  export PATH := $(BIN_OUT):$(PATH)
-else
-# find libleanshared.so
-  TEST_SHARED_LINK_FLAGS := -Wl,-rpath,`lean --print-prefix`/lib/lean
-endif
-
-$(BIN_OUT)/testcpp.so: $(CPP_OBJS) | $(BIN_OUT)
-	$(CXX) -shared -o $@ $^ `leanc -shared --print-ldflags`
-
-$(BIN_OUT)/test: $(LIB_OUT)/libMain.a $(CPP_OBJS) | $(BIN_OUT)
-	$(CXX) -o $@ $^ `leanc -shared --print-ldflags` -lInit_shared -lleanshared_2 -lleanshared_1 -lleanshared $(TEST_SHARED_LINK_FLAGS)
-
-run_test: $(BIN_OUT)/test
-	$^
-
-# also test interpreter; see doc/dev/ffi.md
-$(BIN_OUT)/S.so: $(C_OUT)/Main/S.c $(BIN_OUT)/testcpp.so
-	leanc -shared -o $@ $^
-
-run_interp: $(BIN_OUT)/S.so
-	lean --load-dynlib=$^ --run Main.lean

tests/compiler/foreign/README.md

@@ -1,19 +0,0 @@
-Small project implemented using Lean and C++.
-The C++ file `myfuns.cpp` wraps a C++ object using an `lean_external_object`.
-The file `myfuns.cpp` exposes pure and effectful primitives.
-The file `Main.lean` contains a small Lean program that uses the exported primitives.
-
-Build instructions
-=====
-
-Assuming the Lean `bin/` directory (e.g. from `build/release/stage1`) is in your `PATH`,
-executing `leanmake build/bin/test` will create the executable `build/bin/test`; see the
-`Makefile` for further variants.
-
-The executable `build/bin/test` should produce the output
-```
-30
-hello
-foobla
-world
-```

tests/compiler/foreign/myfuns.cpp

@@ -1,61 +0,0 @@
-#include <stdio.h>
-#include <string>
-#include <lean/lean.h>
-
-struct S {
-    unsigned    m_x;
-    unsigned    m_y;
-    std::string m_s;
-    S(unsigned x, unsigned y, char const * s):m_x(x), m_y(y), m_s(s) {}
-};
-
-static void S_finalize(void * obj) {
-    delete static_cast<S*>(obj);
-}
-
-static void S_foreach(void *, b_lean_obj_arg) {
-    // do nothing since `S` does not contain nested Lean objects
-}
-
-static lean_external_class * g_S_class = nullptr;
-
-static inline lean_object * S_to_lean(S * s) {
-    if (g_S_class == nullptr) {
-        g_S_class = lean_register_external_class(S_finalize, S_foreach);
-    }
-    return lean_alloc_external(g_S_class, s);
-}
-
-static inline S const * to_S(b_lean_obj_arg s) {
-    return static_cast<S *>(lean_get_external_data(s));
-}
-
-extern "C" LEAN_EXPORT lean_object * lean_mk_S(uint32_t x, uint32_t y, b_lean_obj_arg s) {
-    return S_to_lean(new S(x, y, lean_string_cstr(s)));
-}
-
-extern "C" LEAN_EXPORT uint32_t lean_S_add_x_y(b_lean_obj_arg s) {
-    return to_S(s)->m_x + to_S(s)->m_y;
-}
-
-extern "C" LEAN_EXPORT lean_object * lean_S_string(b_lean_obj_arg s) {
-    return lean_mk_string(to_S(s)->m_s.c_str());
-}
-
-static S g_s(0, 0, "");
-
-extern "C" LEAN_EXPORT lean_object * lean_S_global_append(b_lean_obj_arg str, lean_object * /* w */) {
-    g_s.m_s += lean_string_cstr(str);
-    return lean_io_result_mk_ok(lean_box(0));
-}
-
-extern "C" LEAN_EXPORT lean_object * lean_S_global_string(lean_object * /* w */) {
-    return lean_io_result_mk_ok(lean_mk_string(g_s.m_s.c_str()));
-}
-
-extern "C" LEAN_EXPORT lean_object * lean_S_update_global(b_lean_obj_arg s, lean_object * /* w */) {
-    g_s.m_x = to_S(s)->m_x;
-    g_s.m_y = to_S(s)->m_y;
-    g_s.m_s = to_S(s)->m_s;
-    return lean_io_result_mk_ok(lean_box(0));
-}

tests/compiler/test_single.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-source ../common.sh
-
-# First check the C version actually works...
-echo "running C program..."
-rm "./$f.out" || true
-compile_lean_c_backend
-exec_check "./$f.out"
-diff_produced
-
-# Then check the LLVM version
-if lean_has_llvm_support; then
-    echo "running LLVM program..."
-    rm "./$f.out" || true
-    compile_lean_llvm_backend
-    exec_check "./$f.out"
-    diff_produced
-fi

tests/compiler/test_single_interpret.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-source ../common.sh
-
-exec_check lean -Dlinter.all=false --run "$f"
-diff_produced

tests/docparse/.gitattributes

@@ -0,0 +1,4 @@
+# These files should not have line endings translated on Windows
+# because it throws off parser tests.
+*.txt eol=lf
+*.txt.out.expected eol=lf