forked from swiftlang/swift-corelibs-foundation
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[test] Split TestFoundation in smaller tests for CTest reporting.
At the moment, only one test was created for CTest, which would have run all the test in TestFoundation, and would have reported the final result as the test result, but without details of which test have failed. Following the similar case of GTest and gtest_discover_tests, XCTest can be used to list all the tests in the suite, and execute them as invidual CTest. They will report as different tests in the output, and individual failures will be shown. It will be also a little bit more resilient against crashes, which will only affect one test, and not the full suite. At first I was thinking in doing the parsing in some scripting language, but in order to be platform independent, and because the parsing is easy enough, the parsing of the XCTest output is done in CMake.
- Loading branch information
1 parent
e6e0046
commit 569e423
Showing
4 changed files
with
198 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| cmake_policy(PUSH) | ||
| cmake_policy(SET CMP0057 NEW) | ||
|
|
||
| # Automatically add tests with CTest by querying the compiled test executable | ||
| # for available tests. | ||
| # | ||
| # xctest_discover_tests(target | ||
| # [COMMAND command] | ||
| # [WORKING_DIRECTORY dir] | ||
| # [PROPERTIES name1 value1...] | ||
| # [DISCOVERY_TIMEOUT seconds] | ||
| # ) | ||
| # | ||
| # `xctest_discover_tests` sets up a post-build command on the test executable | ||
| # that generates the list of tests by parsing the output from running the test | ||
| # with the `--list-tests` argument. | ||
| # | ||
| # The options are: | ||
| # | ||
| # `target` | ||
| # Specifies the XCTest executable, which must be a known CMake target. CMake | ||
| # will substitute the location of the built executable when running the test. | ||
| # | ||
| # `COMMAND command` | ||
| # Override the command used for the test executable. If you executable is not | ||
| # created with CMake add_executable, you will have to provide a command path. | ||
| # If this option is not provided, the target file of the target is used. | ||
| # | ||
| # `WORKING_DIRECTORY dir` | ||
| # Specifies the directory in which to run the discovered test cases. If this | ||
| # option is not provided, the current binary directory is used. | ||
| # | ||
| # `PROPERTIES name1 value1...` | ||
| # Specifies additional properties to be set on all tests discovered by this | ||
| # invocation of `xctest_discover_tests`. | ||
| # | ||
| # `DISCOVERY_TIMEOUT seconds` | ||
| # Specifies how long (in seconds) CMake will wait for the test to enumerate | ||
| # available tests. If the test takes longer than this, discovery (and your | ||
| # build) will fail. The default is 5 seconds. | ||
| # | ||
| # The inspiration for this is CMake `gtest_discover_tests`. The official | ||
| # documentation might be useful for using this function. Many details of that | ||
| # function has been dropped in the name of simplicity, and others have been | ||
| # improved. | ||
| function(xctest_discover_tests TARGET) | ||
| cmake_parse_arguments( | ||
| "" | ||
| "" | ||
| "COMMAND;WORKING_DIRECTORY;DISCOVERY_TIMEOUT" | ||
| "PROPERTIES" | ||
| ${ARGN} | ||
| ) | ||
|
|
||
| if(NOT _COMMAND) | ||
| set(_COMMAND "$<TARGET_FILE:${TARGET}>") | ||
| endif() | ||
| if(NOT _WORKING_DIRECTORY) | ||
| set(_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) | ||
| endif() | ||
| if(NOT _DISCOVERY_TIMEOUT) | ||
| set(_DISCOVERY_TIMEOUT 5) | ||
| endif() | ||
|
|
||
| set(ctest_file_base ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}) | ||
| set(ctest_include_file "${ctest_file_base}_include.cmake") | ||
| set(ctest_tests_file "${ctest_file_base}_tests.cmake") | ||
|
|
||
| add_custom_command( | ||
| TARGET ${TARGET} POST_BUILD | ||
| BYPRODUCTS "${ctest_tests_file}" | ||
| COMMAND "${CMAKE_COMMAND}" | ||
| -D "TEST_TARGET=${TARGET}" | ||
| -D "TEST_EXECUTABLE=${_COMMAND}" | ||
| -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" | ||
| -D "TEST_PROPERTIES=${_PROPERTIES}" | ||
| -D "CTEST_FILE=${ctest_tests_file}" | ||
| -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" | ||
| -P "${_XCTEST_DISCOVER_TESTS_SCRIPT}" | ||
| VERBATIM | ||
| ) | ||
|
|
||
| file(WRITE "${ctest_include_file}" | ||
| "if(EXISTS \"${ctest_tests_file}\")\n" | ||
| " include(\"${ctest_tests_file}\")\n" | ||
| "else()\n" | ||
| " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" | ||
| "endif()\n" | ||
| ) | ||
|
|
||
| set_property(DIRECTORY | ||
| APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" | ||
| ) | ||
| endfunction() | ||
|
|
||
| set(_XCTEST_DISCOVER_TESTS_SCRIPT | ||
| ${CMAKE_CURRENT_LIST_DIR}/XCTestAddTests.cmake | ||
| ) | ||
|
|
||
| cmake_policy(POP) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| set(properties ${TEST_PROPERTIES}) | ||
| set(script) | ||
| set(tests) | ||
|
|
||
| function(add_command NAME) | ||
| set(_args "") | ||
| foreach(_arg ${ARGN}) | ||
| if(_arg MATCHES "[^-./:a-zA-Z0-9_]") | ||
| set(_args "${_args} [==[${_arg}]==]") | ||
| else() | ||
| set(_args "${_args} ${_arg}") | ||
| endif() | ||
| endforeach() | ||
| set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) | ||
| endfunction() | ||
|
|
||
| if(NOT EXISTS "${TEST_EXECUTABLE}") | ||
| message(FATAL_ERROR | ||
| "Specified test executable does not exist.\n" | ||
| " Path: '${TEST_EXECUTABLE}'" | ||
| ) | ||
| endif() | ||
| # We need to figure out if some environment is needed to run the test listing. | ||
| cmake_parse_arguments("_properties" "" "ENVIRONMENT" "" ${properties}) | ||
| if(_properties_ENVIRONMENT) | ||
| foreach(_env ${_properties_ENVIRONMENT}) | ||
| string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\1" _key "${_env}") | ||
| string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\2" _value "${_env}") | ||
| if(NOT "${_key}" STREQUAL "") | ||
| set(ENV{${_key}} "${_value}") | ||
| endif() | ||
| endforeach() | ||
| endif() | ||
| execute_process( | ||
| COMMAND "${TEST_EXECUTABLE}" --list-tests | ||
| WORKING_DIRECTORY "${TEST_WORKING_DIR}" | ||
| TIMEOUT ${TEST_DISCOVERY_TIMEOUT} | ||
| OUTPUT_VARIABLE output | ||
| ERROR_VARIABLE error_output | ||
| RESULT_VARIABLE result | ||
| ) | ||
| if(NOT ${result} EQUAL 0) | ||
| string(REPLACE "\n" "\n " output "${output}") | ||
| string(REPLACE "\n" "\n " error_output "${error_output}") | ||
| message(FATAL_ERROR | ||
| "Error running test executable.\n" | ||
| " Path: '${TEST_EXECUTABLE}'\n" | ||
| " Result: ${result}\n" | ||
| " Output:\n" | ||
| " ${output}\n" | ||
| " Error:\n" | ||
| " ${error_output}\n" | ||
| ) | ||
| endif() | ||
|
|
||
| string(REPLACE "\n" ";" output "${output}") | ||
|
|
||
| foreach(line ${output}) | ||
| if(line MATCHES "^[ \t]*$") | ||
| continue() | ||
| elseif(line MATCHES "^Listing [0-9]+ tests? in .+:$") | ||
| continue() | ||
| elseif(line MATCHES "^.+\\..+/.+$") | ||
| # TODO: remove non-ASCII characters from module, class and method names | ||
| set(pretty_target "${line}") | ||
| string(REGEX REPLACE "/" "-" pretty_target "${pretty_target}") | ||
| add_command(add_test | ||
| "${pretty_target}" | ||
| "${TEST_EXECUTABLE}" | ||
| "${line}" | ||
| ) | ||
| add_command(set_tests_properties | ||
| "${pretty_target}" | ||
| PROPERTIES | ||
| WORKING_DIRECTORY "${TEST_WORKING_DIR}" | ||
| ${properties} | ||
| ) | ||
| list(APPEND tests "${pretty_target}") | ||
| else() | ||
| message(FATAL_ERROR | ||
| "Error parsing test executable output.\n" | ||
| " Path: '${TEST_EXECUTABLE}'\n" | ||
| " Line: '${line}'" | ||
| ) | ||
| endif() | ||
| endforeach() | ||
|
|
||
| add_command(set "${TARGET}_TESTS" ${tests}) | ||
|
|
||
| file(WRITE "${CTEST_FILE}" "${script}") |