Restore LLVM Testing Infrastructure (microsoft#4012)

* Restore lit and googletest sources from LLVM 3.7.0

This commit just re-adds sources that were removed somewhere along the
way.

* Pull in googletest & googlemock from LLVM 4.0

Googlemock was introduced in LLVM 4.0, and is used by SPIR-V's tests.
This pulls in the LLVM 4.0 version of GoogleTest and GoogleMock to
replace the external submodules.

LLVM's version of GoolgeTest and GoogleMock have some minor extensions
to work better with LIT for error reporting and producing cleaner test
output.

* Remove external googletest

* Fix lit to handle comment in gtest names

This fix came into LLVM with the updated googletest in a977582dead2

* Ignore raw_fd_ostream errors when not closing

There's some odditites with the changes in the filesystem code that
cause this to error sometimes in unit tests. Until we can dedicate time
to looking into the filesystem code, just swallow that error...

* Fix bot failure

Missed an option change.

* Fixing MSVC build failure

* A cleaner build fix

This should address issues with VS 2019 in a cleaner way.

* Fix build

CMakeLists.txt

@@ -100,6 +100,7 @@ option(HLSL_SUPPORT_QUERY_GIT_COMMIT_INFO "Supports querying Git commit info." O
 if ( HLSL_SUPPORT_QUERY_GIT_COMMIT_INFO )
   add_definitions(-DSUPPORT_QUERY_GIT_COMMIT_INFO)
 endif()
+
 # adjust link option to enable debugging from kernel mode; not compatible with incremental linking
 if(NOT CMAKE_VERSION VERSION_LESS "3.13" AND WIN32)
   add_link_options(/DEBUGTYPE:CV,FIXUP,PDATA /INCREMENTAL:NO)
@@ -651,7 +652,7 @@ else()
   endif()
 endif()
 
-if(LLVM_INCLUDE_TESTS)
+if(LLVM_INCLUDE_TESTS OR HLSL_INCLUDE_TESTS OR SPIRV_BUILD_TESTS) # HLSL Change
   add_subdirectory(utils/unittest)
 endif()
 
@@ -756,4 +757,4 @@ endif (HLSL_OFFICIAL_BUILD)
 
 if (NOT "${DXC_CMAKE_ENDS_INCLUDE}" STREQUAL "")
   include(${DXC_CMAKE_ENDS_INCLUDE})
-endif()
+endif()

cmake/modules/AddLLVM.cmake

@@ -770,8 +770,8 @@ function(add_unittest test_suite test_name)
     set(EXCLUDE_FROM_ALL ON)
   endif()
 
-  include_directories(${DXC_GTEST_DIR}/googletest/include) # SPIRV change
-  include_directories(${DXC_GTEST_DIR}/googlemock/include) # SPIRV change
+  include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include)
+  include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock/include) # HLSL Change - Pulled in from LLVM 4.0
   if (NOT LLVM_ENABLE_THREADS)
     list(APPEND LLVM_COMPILE_DEFINITIONS GTEST_HAS_PTHREAD=0)
   endif ()
@@ -787,7 +787,7 @@ function(add_unittest test_suite test_name)
   set_output_directory(${test_name} ${outdir} ${outdir})
   target_link_libraries(${test_name}
     gtest
-    # gtest_main # SPIRV change
+    gtest_main
     LLVMSupport # gtest needs it for raw_ostream.
     )
 

external/CMakeLists.txt

@@ -2,6 +2,11 @@
 set(DXC_EXTERNAL_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
     CACHE STRING "Root location of all external projects")
 
+# We need to match this setting across everything we link together
+if (NOT HLSL_ENABLE_DEBUG_ITERATORS)
+  add_definitions(/D_ITERATOR_DEBUG_LEVEL=0)
+endif (NOT HLSL_ENABLE_DEBUG_ITERATORS)
+
 # Enabling SPIR-V codegen requires SPIRV-Headers for spirv.hpp and
 # SPIRV-Tools for SPIR-V disassembling functionality.
 if (${ENABLE_SPIRV_CODEGEN})
@@ -47,21 +52,6 @@ if (${ENABLE_SPIRV_CODEGEN})
     set_property(TARGET ${target} PROPERTY FOLDER "External dependencies")
   endforeach()
 
-  # We need GoogleTest for unit and SPIR-V codegen testing.
-  if (${SPIRV_BUILD_TESTS})
-    set(DXC_GTEST_DIR "${DXC_EXTERNAL_ROOT_DIR}/googletest"
-        CACHE STRING "Location of GoogleTest source")
-    if (NOT TARGET gtest)
-      if (IS_DIRECTORY ${DXC_GTEST_DIR})
-        # Configure googletest
-        include(GTestConfig.cmake)
-      endif()
-      if (NOT TARGET gtest)
-        message(FATAL_ERROR "GoogleTest was not found - required for SPIR-V codegen")
-      endif()
-    endif()
-  endif()
-
   # We need Effcee for SPIR-V codegen testing.
   if (${SPIRV_BUILD_TESTS})
     # re2 is needed since it's required by effcee.

lib/Support/raw_ostream.cpp

@@ -603,7 +603,7 @@ raw_fd_ostream::~raw_fd_ostream() {
   // to avoid report_fatal_error calls should check for errors with
   // has_error() and clear the error flag with clear_error() before
   // destructing raw_ostream objects which may have errors.
-  if (has_error())
+  if (has_error() && ShouldClose) // HLSL Change - ignore error on !ShouldClose
     report_fatal_error("IO failure on output stream.", /*GenCrashDiag=*/false);
 }
 

tools/clang/unittests/HLSLTestLib/CMakeLists.txt

@@ -18,8 +18,9 @@ add_clang_library(HLSLTestLib
   DxcTestUtils.cpp
   FileCheckForTest.cpp
 )
-include_directories(${DXC_GTEST_DIR}/googletest/include)
-include_directories(${DXC_GTEST_DIR}/googlemock/include)
+include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include)
+include_directories(${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock/include)
+
 endif(WIN32)
 
 add_dependencies(HLSLTestLib TablegenHLSLOptions)

utils/lit/MANIFEST.in

@@ -0,0 +1,9 @@
+include TODO lit.py
+recursive-include tests *
+recursive-include examples *
+global-exclude *pyc
+global-exclude *~
+prune tests/Output
+prune tests/*/Output
+prune tests/*/*/Output
+prune tests/*/*/*/Output

utils/lit/README.txt

@@ -0,0 +1,8 @@
+===============================
+ lit - A Software Testing Tool
+===============================
+
+lit is a portable tool for executing LLVM and Clang style test suites,
+summarizing their results, and providing indication of failures. lit is designed
+to be a lightweight testing tool with as simple a user interface as possible.
+

utils/lit/TODO

@@ -0,0 +1,164 @@
+================
+ lit TODO Items
+================
+
+Infrastructure
+==============
+
+1. Change to always load suites, then resolve command line arguments?
+
+  Currently we expect each input argument to be a path on disk; we do a
+  recursive search to find the test suite for each item, but then we only do a
+  local search based at the input path to find tests. Additionally, for any path
+  that matches a file on disk we explicitly construct a test instance (bypassing
+  the formats on discovery implementation).
+
+  This has a couple problems:
+
+  * The test format doesn't have control over the test instances that result
+    from file paths.
+
+  * It isn't possible to specify virtual tests as inputs. For example, it is not
+    possible to specify an individual subtest to run with the googletest format.
+
+  * The test format doesn't have full control over the discovery of tests in
+    subdirectories.
+
+  Instead, we should move to a model whereby first all of the input specifiers
+  are resolved to test suites, and then the resolution of the input specifier is
+  delegated to each test suite. This could take a couple forms:
+
+  * We could resolve to test suites, then fully load each test suite, then have
+    a fixed process to map input specifiers to tests in the test suite
+    (presumably based on path-in-suite derivations). This has the benefit of
+    being consistent across all test formats, but the downside of requiring
+    loading the entire test suite.
+
+  * We could delegate all of the resolution of specifiers to the test
+    suite. This would allow formats that anticipate large test suites to manage
+    their own resolution for better performance. We could provide a default
+    resolution strategy that was similar to what we do now (start at subpaths
+    for directories, but allow the test format control over what happens for
+    individual tests).
+
+2. Consider move to identifying all tests by path-to-test-suite and then path to
+   subtest, and don't use test suite names.
+
+  Currently the test suite name is presented as part of test names, but it has
+  no other useful function, and it is something that has to be skipped over to
+  cut-and-paste a name to subsequently use to rerun a test. If we just
+  represented each test suite by the path to its suite, then it would allow more
+  easy cut-and-paste of the test output lines. This has the downside that the
+  lines might get rather long.
+
+3. Allow 'lit' driver to cooperate with test formats and suites to add options
+   (or at least sanitize accepted params).
+
+  We have started to use the --params method more and more extensively, and it is
+  cumbersome and error prone. Additionally, there are currently various options
+  ``lit`` honors that should more correctly be specified as belonging to the
+  ShTest test format.
+
+  It would be really nice if we could allow test formats and test suites to add
+  their own options to be parsed. The difficulty here, of course, is that we
+  don't know what test formats or test suites are in use until we have parsed the
+  input specifiers. For test formats we could ostensibly require all the possible
+  formats to be registered in order to have options, but for test suites we would
+  certainly have to load the suite before we can query it for what options it
+  understands.
+
+  That leaves us with the following options:
+
+  * Currently we could almost get away with parsing the input specifiers without
+    having done option parsing first (the exception is ``--config-prefix``) but
+    that isn't a very extensible design.
+
+  * We could make a distinction in the command line syntax for test format and
+    test suite options. For example, we could require something like::
+
+      lit -j 1 -sv input-specifier -- --some-format-option
+
+    which would be relatively easy to implement with optparser (I think).
+
+  * We could allow fully interspersed arguments by first extracting the options
+    lit knows about and parsing them, then dispatching the remainder to the
+    formats. This seems the most convenient for users, who are unlikely to care
+    about (or even be aware of) the distinction between the generic lit
+    infrastructure and format or suite specific options.
+
+4. Eliminate duplicate execution models for ShTest tests.
+
+  Currently, the ShTest format uses tests written with shell-script like syntax,
+  and executes them in one of two ways. The first way is by converting them into
+  a bash script and literally executing externally them using bash. The second
+  way is through the use of an internal shell parser and shell execution code
+  (built on the subprocess module). The external execution mode is used on most
+  Unix systems that have bash, the internal execution mode is used on Windows.
+
+  Having two ways to do the same thing is error prone and leads to unnecessary
+  complexity in the testing environment. Additionally, because the mode that
+  converts scripts to bash doesn't try and validate the syntax, it is possible
+  to write tests that use bash shell features unsupported by the internal
+  shell. Such tests won't work on Windows but this may not be obvious to the
+  developer writing the test.
+
+  Another limitation is that when executing the scripts externally, the ShTest
+  format has no idea which commands fail, or what output comes from which
+  commands, so this limits how convenient the output of ShTest failures can be
+  and limits other features (for example, knowing what temporary files were
+  written).
+
+  We should eliminate having two ways of executing the same tests to reduce
+  platform differences and make it easier to develop new features in the ShTest
+  module. This is currently blocked on:
+
+  * The external execution mode is faster in some situations, because it avoids
+    being bottlenecked on the GIL. This can hopefully be obviated simply by
+    using --use-processes.
+
+  * Some tests in LLVM/Clang are explicitly disabled with the internal shell
+    (because they use features specific to bash). We would need to rewrite these
+    tests, or add additional features to the internal shell handling to allow
+    them to pass.
+
+5. Consider changing core to support setup vs. execute distinction.
+
+  Many of the existing test formats are cleanly divided into two phases, once
+  parses the test format and extracts XFAIL and REQUIRES information, etc., and
+  the other code actually executes the test.
+
+  We could make this distinction part of the core infrastructure and that would
+  enable a couple things:
+
+  * The REQUIREs handling could be lifted to the core, which is nice.
+
+  * This would provide a clear place to insert subtest support, because the
+    setup phase could be responsible for providing subtests back to the
+    core. That would provide part of the infrastructure to parallelize them, for
+    example, and would probably interact well with other possible features like
+    parameterized tests.
+
+  * This affords a clean implementation of --no-execute.
+
+  * One possible downside could be for test formats that cannot determine their
+    subtests without having executed the test. Supporting such formats would
+    either force the test to actually be executed in the setup stage (which
+    might be ok, as long as the API was explicitly phrased to support that), or
+    would mean we are forced into supporting subtests as return values from the
+    execute phase.
+
+  Any format can just keep all of its code in execute, presumably, so the only
+  cost of implementing this is its impact on the API and futures changes.
+
+
+Miscellaneous
+=============
+
+* Move temp directory name into local test config.
+
+* Support valgrind in all configs, and LLVM style valgrind.
+
+* Support a timeout / ulimit.
+
+* Create an explicit test suite object (instead of using the top-level
+  TestingConfig object).

utils/lit/examples/README.txt

@@ -0,0 +1,7 @@
+==============
+ lit Examples
+==============
+
+This directory contains examples of 'lit' test suite configurations. The test
+suites they define can be run with 'lit examples/example-name', for more details
+see the README in each example.

utils/lit/examples/many-tests/README.txt

@@ -0,0 +1,10 @@
+========================
+ Many Tests lit Example
+========================
+
+This directory contains a trivial lit test suite configuration that defines a
+custom test format which just generates a large (N=10000) number of tests that
+do a small amount of work in the Python test execution code.
+
+This test suite is useful for testing the performance of lit on large numbers of
+tests.

utils/lit/examples/many-tests/lit.cfg

@@ -0,0 +1,23 @@
+# -*- Python -*-
+
+from lit import Test
+
+class ManyTests(object):
+    def __init__(self, N=10000):
+        self.N = N
+
+    def getTestsInDirectory(self, testSuite, path_in_suite,
+                            litConfig, localConfig):
+        for i in range(self.N):
+            test_name = 'test-%04d' % (i,)
+            yield Test.Test(testSuite, path_in_suite + (test_name,),
+                            localConfig)
+
+    def execute(self, test, litConfig):
+        # Do a "non-trivial" amount of Python work.
+        sum = 0
+        for i in range(10000):
+            sum += i
+        return Test.PASS,''
+
+config.test_format = ManyTests()

utils/lit/lit.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+if __name__=='__main__':
+    import lit
+    lit.main()

utils/lit/lit/ExampleTests.ObjDir/lit.site.cfg

@@ -0,0 +1,15 @@
+# -*- Python -*-
+
+# Site specific configuration file.
+#
+# Typically this will be generated by the build system to automatically set
+# certain configuration variables which cannot be autodetected, so that 'lit'
+# can easily be used on the command line.
+
+import os
+
+# Preserve the obj_root, for use by the main lit.cfg.
+config.example_obj_root = os.path.dirname(__file__)
+
+lit.load_config(config, os.path.join(config.test_source_root,
+                                     'lit.cfg'))