Merge branch 'test-RunCMake' into object-library

Tests/CMakeCommands/CMakeLists.txt

@@ -1,11 +0,0 @@
-macro(add_CMakeCommands_test test)
-  add_test(CMakeCommands.${test} ${CMAKE_CMAKE_COMMAND}
-    -DCMake_SOURCE_DIR=${CMake_SOURCE_DIR} # TODO: Remove
-    -Ddir=${CMAKE_CURRENT_BINARY_DIR}/${test}
-    -Dgen=${CMAKE_TEST_GENERATOR}
-    -P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/test.cmake"
-    )
-endmacro()
-
-add_CMakeCommands_test(build_command)
-add_CMakeCommands_test(find_package)

Tests/CMakeCommands/build_command/test.cmake

@@ -1,86 +0,0 @@
-if(NOT DEFINED CMake_SOURCE_DIR)
-  message(FATAL_ERROR "CMake_SOURCE_DIR not defined")
-endif()
-
-if(NOT DEFINED dir)
-  message(FATAL_ERROR "dir not defined")
-endif()
-
-if(NOT DEFINED gen)
-  message(FATAL_ERROR "gen not defined")
-endif()
-
-message(STATUS "CTEST_FULL_OUTPUT (Avoid ctest truncation of output)")
-
-# Run cmake:
-#
-function(run_cmake build_dir extra_args expected_result expected_output expected_error)
-  message(STATUS "run_cmake build_dir='${build_dir}' extra_args='${extra_args}'")
-
-  # Ensure build_dir exists:
-  #
-  execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${build_dir})
-
-  # Run cmake:
-  #
-  execute_process(COMMAND ${CMAKE_COMMAND}
-    ${extra_args}
-    -G ${gen} ${CMake_SOURCE_DIR}/Tests/CMakeCommands/build_command
-    RESULT_VARIABLE result
-    OUTPUT_VARIABLE stdout
-    ERROR_VARIABLE stderr
-    WORKING_DIRECTORY ${build_dir}
-    )
-
-  message(STATUS "result='${result}'")
-  message(STATUS "stdout='${stdout}'")
-  message(STATUS "stderr='${stderr}'")
-  message(STATUS "")
-
-  # Verify result and output match expectations:
-  #
-  if("0" STREQUAL "${expected_result}")
-    if(NOT "${result}" STREQUAL "0")
-      message(FATAL_ERROR
-        "error: result='${result}' is non-zero and different than expected_result='${expected_result}'")
-    endif()
-  else()
-    if("${result}" STREQUAL "0")
-      message(FATAL_ERROR
-        "error: result='${result}' is zero and different than expected_result='${expected_result}'")
-    endif()
-  endif()
-
-  foreach(e ${expected_output})
-    if(NOT stdout MATCHES "${e}")
-      message(FATAL_ERROR
-        "error: stdout does not match expected_output item e='${e}'")
-    else()
-      message(STATUS "info: stdout matches '${e}'")
-    endif()
-  endforeach()
-
-  foreach(e ${expected_error})
-    if(NOT stderr MATCHES "${e}")
-      message(FATAL_ERROR
-        "error: stderr does not match expected_error item e='${e}'")
-    else()
-      message(STATUS "info: stderr matches '${e}'")
-    endif()
-  endforeach()
-
-  message(STATUS "result, stdout and stderr match all expectations: test passes")
-  message(STATUS "")
-endfunction()
-
-
-# Expect this case to succeed:
-run_cmake("${dir}/b1" "" 0
-  "Build files have been written to:"
-  "skipping cases 1, 2 and 3 because TEST_ERROR_CONDITIONS is OFF")
-
-
-# Expect this one to fail:
-run_cmake("${dir}/b2" "-DTEST_ERROR_CONDITIONS:BOOL=ON" 1
-  "Configuring incomplete, errors occurred!"
-  "build_command requires at least one argument naming a CMake variable;build_command unknown argument ")

Tests/CMakeCommands/find_package/CMakeLists.txt

@@ -1,3 +0,0 @@
-cmake_minimum_required(VERSION 2.8)
-project(${TEST} NONE)
-include(${TEST}.cmake)

Tests/CMakeLists.txt

@@ -53,7 +53,7 @@ IF(BUILD_TESTING)
 
   ADD_SUBDIRECTORY(CMakeLib)
   ADD_SUBDIRECTORY(CMakeOnly)
-  ADD_SUBDIRECTORY(CMakeCommands)
+  ADD_SUBDIRECTORY(RunCMake)
 
   ADD_SUBDIRECTORY(FindPackageModeMakefileTest)
 

Tests/README

@@ -16,10 +16,15 @@ your test to the test runs.
 This includes tests that will build something using try_compile() and friends,
 but nothing that expects add_executable(), add_library(), or add_test() to run.
 
-If this matches your test you should put it into the Tests/CMakeOnly/ directory.
-Create a subdirectory named like your test and write the CMakeLists.txt you
-need into that subdirectory. Use the add_CMakeOnly_test() macro from
-Tests/CMakeOnly/CMakeLists.txt to add your test to the test runs.
+If the test configures the project only once and it must succeed then put it
+into the Tests/CMakeOnly/ directory.  Create a subdirectory named like your
+test and write the CMakeLists.txt you need into that subdirectory. Use the
+add_CMakeOnly_test() macro from Tests/CMakeOnly/CMakeLists.txt to add your
+test to the test runs.
+
+If the test configures the project with multiple variations and verifies
+success or failure each time then put it into the Tests/RunCMake/ directory.
+Read the instructions in Tests/RunCMake/CMakeLists.txt to add a test.
 
 3. If you are testing something from the Modules directory
 

Tests/RunCMake/CMakeLists.txt

@@ -0,0 +1,44 @@
+# This directory contains tests that run CMake to configure a project
+# but do not actually build anything.  To add a test:
+#
+# 1.) Add a subdirectory named for the test.
+#
+# 2.) Call add_RunCMake_test and pass the test directory name.
+#
+# 3.) Create a RunCMakeTest.cmake script in the directory containing
+#   include(RunCMake)
+#   run_cmake(SubTest1)
+#   ...
+#   run_cmake(SubTestN)
+# where SubTest1..SubTestN are sub-test names each corresponding to
+# an independent CMake run and project configuration.
+#
+# 3.) Create a CMakeLists.txt file in the directory containing
+#   cmake_minimum_required(...)
+#   project(${RunCMake_TEST} NONE) # or languages needed
+#   include(${RunCMake_TEST}.cmake)
+# where "${RunCMake_TEST}" is literal.  A value for RunCMake_TEST
+# will be passed to CMake by the run_cmake macro when running each
+# sub-test.
+#
+# 4.) Create a <SubTest>.cmake file for each sub-test named above
+# containing the actual test code.  Optionally create files
+# containing expected test results:
+#   <SubTest>-result.txt = Process result expected if not "0"
+#   <SubTest>-stdout.txt = Regex matching expected stdout content
+#   <SubTest>-stderr.txt = Regex matching expected stderr content
+# Note that trailing newlines will be stripped from actual test
+# output before matching against the stdout and stderr expressions.
+
+macro(add_RunCMake_test test)
+  add_test(RunCMake.${test} ${CMAKE_CMAKE_COMMAND}
+    -DCMAKE_MODULE_PATH=${CMAKE_CURRENT_SOURCE_DIR}
+    -DRunCMake_GENERATOR=${CMAKE_TEST_GENERATOR}
+    -DRunCMake_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/${test}
+    -DRunCMake_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/${test}
+    -P "${CMAKE_CURRENT_SOURCE_DIR}/${test}/RunCMakeTest.cmake"
+    )
+endmacro()
+
+add_RunCMake_test(build_command)
+add_RunCMake_test(find_package)

Tests/CMakeCommands/find_package/test.cmake → Tests/RunCMake/RunCMake.cmake

@@ -1,15 +1,16 @@
-if(NOT DEFINED dir)
-  message(FATAL_ERROR "dir not defined")
-endif()
-
-if(NOT DEFINED gen)
-  message(FATAL_ERROR "gen not defined")
-endif()
+foreach(arg
+    RunCMake_GENERATOR
+    RunCMake_SOURCE_DIR
+    RunCMake_BINARY_DIR
+    )
+  if(NOT DEFINED ${arg})
+    message(FATAL_ERROR "${arg} not given!")
+  endif()
+endforeach()
 
-# TODO: Generalize this for other tests.
-function(run_test test)
-  set(top_src "${CMAKE_CURRENT_LIST_DIR}")
-  set(top_bin "${dir}")
+function(run_cmake test)
+  set(top_src "${RunCMake_SOURCE_DIR}")
+  set(top_bin "${RunCMake_BINARY_DIR}")
   if(EXISTS ${top_src}/${test}-result.txt)
     file(READ ${top_src}/${test}-result.txt expect_result)
     string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
@@ -29,7 +30,8 @@ function(run_test test)
   file(REMOVE_RECURSE "${binary_dir}")
   file(MAKE_DIRECTORY "${binary_dir}")
   execute_process(
-    COMMAND ${CMAKE_COMMAND} "${source_dir}" -G "${gen}" -DTEST=${test}
+    COMMAND ${CMAKE_COMMAND} "${source_dir}"
+              -G "${RunCMake_GENERATOR}" -DRunCMake_TEST=${test}
     WORKING_DIRECTORY "${binary_dir}"
     OUTPUT_VARIABLE actual_stdout
     ERROR_VARIABLE actual_stderr
@@ -65,16 +67,3 @@ function(run_test test)
     message(STATUS "${test} - PASSED")
   endif()
 endfunction()
-
-run_test(MissingNormal)
-run_test(MissingNormalRequired)
-run_test(MissingNormalVersion)
-run_test(MissingNormalWarnNoModuleOld)
-run_test(MissingNormalWarnNoModuleNew)
-run_test(MissingModule)
-run_test(MissingModuleRequired)
-run_test(MissingConfig)
-run_test(MissingConfigOneName)
-run_test(MissingConfigRequired)
-run_test(MissingConfigVersion)
-run_test(MixedModeOptions)

Tests/CMakeCommands/build_command/CMakeLists.txt → Tests/RunCMake/build_command/CMakeLists.txt

@@ -1,3 +1,7 @@
+cmake_minimum_required(VERSION 2.8)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)
+
 # This CMakeLists file is *sometimes expected* to result in a configure error.
 #
 # expect this to succeed:
@@ -12,12 +16,9 @@
 # ...even purposefully calling it with known-bad argument lists to cover
 # error handling code.
 #
-cmake_minimum_required(VERSION 2.8)
-project(test_build_command)
 
 set(cmd "initial")
 
-message("CTEST_FULL_OUTPUT")
 message("0. begin")
 
 if(TEST_ERROR_CONDITIONS)

Tests/RunCMake/build_command/ErrorsOFF-stderr.txt

@@ -0,0 +1 @@
+skipping cases 1, 2 and 3 because TEST_ERROR_CONDITIONS is OFF

Tests/RunCMake/build_command/ErrorsOFF-stdout.txt

@@ -0,0 +1 @@
+Build files have been written to:

Tests/RunCMake/build_command/ErrorsOFF.cmake

@@ -0,0 +1 @@
+set(TEST_ERROR_CONDITIONS OFF)

Tests/RunCMake/build_command/ErrorsON-stderr.txt

@@ -0,0 +1,12 @@
+CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
+  build_command requires at least one argument naming a CMake variable
+
++
+1. cmd='initial'
+CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
+  build_command unknown argument "BOGUS"
+
++
+2. cmd='initial'
+CMake Error at CMakeLists.txt:[0-9]+ \(build_command\):
+  build_command unknown argument "STUFF"

Tests/RunCMake/build_command/ErrorsON-stdout.txt

@@ -0,0 +1 @@
+Configuring incomplete, errors occurred!

Tests/RunCMake/build_command/ErrorsON.cmake

@@ -0,0 +1 @@
+set(TEST_ERROR_CONDITIONS ON)

Tests/RunCMake/build_command/RunCMakeTest.cmake

@@ -0,0 +1,4 @@
+include(RunCMake)
+
+run_cmake(ErrorsOFF)
+run_cmake(ErrorsON)

Tests/RunCMake/find_package/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 2.8)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/find_package/MixedModeOptions-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/find_package/RunCMakeTest.cmake

@@ -0,0 +1,14 @@
+include(RunCMake)
+
+run_cmake(MissingNormal)
+run_cmake(MissingNormalRequired)
+run_cmake(MissingNormalVersion)
+run_cmake(MissingNormalWarnNoModuleOld)
+run_cmake(MissingNormalWarnNoModuleNew)
+run_cmake(MissingModule)
+run_cmake(MissingModuleRequired)
+run_cmake(MissingConfig)
+run_cmake(MissingConfigOneName)
+run_cmake(MissingConfigRequired)
+run_cmake(MissingConfigVersion)
+run_cmake(MixedModeOptions)