VS: Select and save a VS 2017 instance persistently

Visual Studio 2017 supports multiple instances installed on a single
machine.  We use the Visual Studio Installer tool to enumerate instances
and select one.  Once we select an instance for a given build tree, save
the result in `CMAKE_GENERATOR_INSTANCE` so we can re-configure the tree
with the same instance on future re-runs of CMake.

Fixes: #17268

Help/generator/Visual Studio 15 2017.rst

@@ -19,13 +19,17 @@ Instance Selection
 ^^^^^^^^^^^^^^^^^^
 
 VS 2017 supports multiple installations on the same machine.
-CMake queries the Visual Studio Installer to locate VS instances.
-If more than one instance is installed we do not define which one
-is chosen by default.  If the ``VS150COMNTOOLS`` environment variable
-is set and points to the ``Common7/Tools`` directory within one of
-the instances, that instance will be used.  The environment variable
-must remain consistently set whenever CMake is re-run within a given
-build tree.
+The :variable:`CMAKE_GENERATOR_INSTANCE` variable may be set as a
+cache entry containing the absolute path to a Visual Studio instance.
+If the value is not specified explicitly by the user or a toolchain file,
+CMake queries the Visual Studio Installer to locate VS instances, chooses
+one, and sets the variable as a cache entry to hold the value persistently.
+
+When CMake first chooses an instance, if the ``VS150COMNTOOLS`` environment
+variable is set and points to the ``Common7/Tools`` directory within
+one of the instances, that instance will be used.  Otherwise, if more
+than one instance is installed we do not define which one is chosen
+by default.
 
 Toolset Selection
 ^^^^^^^^^^^^^^^^^

Help/release/dev/generator-instance.rst

@@ -3,5 +3,6 @@ generator-instance
 
 * A :variable:`CMAKE_GENERATOR_INSTANCE` variable was introduced
   to hold the selected instance of the generator's corresponding
-  native tools if multiple are available.  Currently no generators
-  actually use this, but the infrastructure is in place.
+  native tools if multiple are available.  This is used by the
+  :generator:`Visual Studio 15 2017` generator to hold the
+  selected instance of Visual Studio persistently.

Help/variable/CMAKE_GENERATOR_INSTANCE.rst

@@ -17,6 +17,8 @@ for this variable, changing the value has undefined behavior.
 
 Instance specification is supported only on specific generators:
 
-* None
+* For the :generator:`Visual Studio 15 2017` generator (and above)
+  this specifies the absolute path to the VS installation directory
+  of the selected VS instance.
 
 See native build system documentation for allowed instance values.

Source/cmCMakeHostSystemInformationCommand.cxx

@@ -8,6 +8,9 @@
 #include "cmsys/SystemInformation.hxx"
 
 #if defined(_WIN32)
+#include "cmAlgorithms.h"
+#include "cmGlobalGenerator.h"
+#include "cmGlobalVisualStudio15Generator.h"
 #include "cmSystemTools.h"
 #include "cmVSSetupHelper.h"
 #define HAVE_VS_SETUP_HELPER
@@ -127,6 +130,17 @@ bool cmCMakeHostSystemInformationCommand::GetValue(
     value = this->ValueToString(info.GetOSPlatform());
 #ifdef HAVE_VS_SETUP_HELPER
   } else if (key == "VS_15_DIR") {
+    // If generating for the VS 15 IDE, use the same instance.
+    cmGlobalGenerator* gg = this->Makefile->GetGlobalGenerator();
+    if (cmHasLiteralPrefix(gg->GetName(), "Visual Studio 15 ")) {
+      cmGlobalVisualStudio15Generator* vs15gen =
+        static_cast<cmGlobalVisualStudio15Generator*>(gg);
+      if (vs15gen->GetVSInstance(value)) {
+        return true;
+      }
+    }
+
+    // Otherwise, find a VS 15 instance ourselves.
     cmVSSetupAPIHelper vsSetupAPIHelper;
     if (vsSetupAPIHelper.GetVSInstanceInfo(value)) {
       cmSystemTools::ConvertToUnixSlashes(value);

Source/cmGlobalVisualStudio15Generator.cxx

@@ -111,6 +111,53 @@ void cmGlobalVisualStudio15Generator::WriteSLNHeader(std::ostream& fout)
   }
 }
 
+bool cmGlobalVisualStudio15Generator::SetGeneratorInstance(
+  std::string const& i, cmMakefile* mf)
+{
+  if (!i.empty()) {
+    if (!this->vsSetupAPIHelper.SetVSInstance(i)) {
+      std::ostringstream e;
+      /* clang-format off */
+      e <<
+        "Generator\n"
+        "  " << this->GetName() << "\n"
+        "could not find specified instance of Visual Studio:\n"
+        "  " << i;
+      /* clang-format on */
+      mf->IssueMessage(cmake::FATAL_ERROR, e.str());
+      return false;
+    }
+  }
+
+  std::string vsInstance;
+  if (!this->vsSetupAPIHelper.GetVSInstanceInfo(vsInstance)) {
+    std::ostringstream e;
+    /* clang-format off */
+    e <<
+      "Generator\n"
+      "  " << this->GetName() << "\n"
+      "could not find any instance of Visual Studio.\n";
+    /* clang-format on */
+    mf->IssueMessage(cmake::FATAL_ERROR, e.str());
+    return false;
+  }
+
+  // Save the selected instance persistently.
+  std::string genInstance = mf->GetSafeDefinition("CMAKE_GENERATOR_INSTANCE");
+  if (vsInstance != genInstance) {
+    this->CMakeInstance->AddCacheEntry(
+      "CMAKE_GENERATOR_INSTANCE", vsInstance.c_str(),
+      "Generator instance identifier.", cmStateEnums::INTERNAL);
+  }
+
+  return true;
+}
+
+bool cmGlobalVisualStudio15Generator::GetVSInstance(std::string& dir) const
+{
+  return vsSetupAPIHelper.GetVSInstanceInfo(dir);
+}
+
 bool cmGlobalVisualStudio15Generator::InitializeWindows(cmMakefile* mf)
 {
   // If the Win 8.1 SDK is installed then we can select a SDK matching

Source/cmGlobalVisualStudio15Generator.h

@@ -27,6 +27,11 @@ public:
   virtual void WriteSLNHeader(std::ostream& fout);
 
   virtual const char* GetToolsVersion() { return "15.0"; }
+
+  bool SetGeneratorInstance(std::string const& i, cmMakefile* mf) override;
+
+  bool GetVSInstance(std::string& dir) const;
+
 protected:
   bool InitializeWindows(cmMakefile* mf) override;
   virtual bool SelectWindowsStoreToolset(std::string& toolset) const;

Source/cmVSSetupHelper.cxx

@@ -273,13 +273,6 @@ bool cmVSSetupAPIHelper::EnumerateAndChooseVSInstance()
   if (cmSystemTools::GetEnv("VS150COMNTOOLS", envVSCommonToolsDir)) {
     cmSystemTools::ConvertToUnixSlashes(envVSCommonToolsDir);
   }
-  // FIXME: If the environment variable value changes between runs
-  // of CMake within a given build tree the results are not defined.
-  // Instead we should save a CMAKE_GENERATOR_INSTANCE value in the cache
-  // (similar to CMAKE_GENERATOR_TOOLSET) to hold it persistently.
-  // Unfortunately doing so will require refactoring elsewhere in
-  // order to make sure the value is available in time to create
-  // the generator.
 
   std::vector<VSInstanceInfo> vecVSInstances;
   SmartCOMPtr<IEnumSetupInstances> enumInstances = NULL;

Tests/RunCMake/GeneratorInstance/DefaultInstance.cmake

@@ -0,0 +1,13 @@
+if("x${CMAKE_GENERATOR_INSTANCE}" STREQUAL "x")
+  message(FATAL_ERROR "CMAKE_GENERATOR_INSTANCE is empty but should have a value.")
+elseif("x${CMAKE_GENERATOR_INSTANCE}" MATCHES [[\\]])
+  message(FATAL_ERROR
+    "CMAKE_GENERATOR_INSTANCE is\n"
+    "  ${CMAKE_GENERATOR_INSTANCE}\n"
+    "which contains a backslash.")
+elseif(NOT IS_DIRECTORY "${CMAKE_GENERATOR_INSTANCE}")
+  message(FATAL_ERROR
+    "CMAKE_GENERATOR_INSTANCE is\n"
+    "  ${CMAKE_GENERATOR_INSTANCE}\n"
+    "which is not an existing directory.")
+endif()

Tests/RunCMake/GeneratorInstance/MissingInstance-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/GeneratorInstance/MissingInstance-stderr.txt

@@ -0,0 +1,8 @@
+CMake Error at CMakeLists.txt:[0-9]+ \(project\):
+  Generator
+
+    .*
+
+  could not find specified instance of .*:
+
+    .*/Tests/RunCMake/GeneratorInstance/instance_does_not_exist$

Tests/RunCMake/GeneratorInstance/MissingInstance-toolchain.cmake

@@ -0,0 +1 @@
+set(CMAKE_GENERATOR_INSTANCE "${CMAKE_CURRENT_LIST_DIR}/instance_does_not_exist")

Tests/RunCMake/GeneratorInstance/MissingInstance.cmake

@@ -0,0 +1 @@
+message(FATAL_ERROR "This should not be reached!")

Tests/RunCMake/GeneratorInstance/MissingInstanceToolchain-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/GeneratorInstance/MissingInstanceToolchain-stderr.txt

@@ -0,0 +1,8 @@
+CMake Error at CMakeLists.txt:[0-9]+ \(project\):
+  Generator
+
+    .*
+
+  could not find specified instance of .*:
+
+    .*/Tests/RunCMake/GeneratorInstance/instance_does_not_exist$

Tests/RunCMake/GeneratorInstance/MissingInstanceToolchain.cmake

@@ -0,0 +1 @@
+message(FATAL_ERROR "This should not be reached!")

Tests/RunCMake/GeneratorInstance/RunCMakeTest.cmake

@@ -1,11 +1,22 @@
 include(RunCMake)
 
-set(RunCMake_GENERATOR_INSTANCE "")
-run_cmake(NoInstance)
+if("${RunCMake_GENERATOR}" MATCHES "^Visual Studio 1[56789]")
+  set(RunCMake_GENERATOR_INSTANCE "")
+  run_cmake(DefaultInstance)
 
-set(RunCMake_GENERATOR_INSTANCE "Bad Instance")
-run_cmake(BadInstance)
+  set(RunCMake_GENERATOR_INSTANCE "${RunCMake_SOURCE_DIR}/instance_does_not_exist")
+  run_cmake(MissingInstance)
+  set(RunCMake_TEST_OPTIONS -DCMAKE_TOOLCHAIN_FILE=${RunCMake_SOURCE_DIR}/MissingInstance-toolchain.cmake)
+  run_cmake(MissingInstanceToolchain)
+  unset(RunCMake_TEST_OPTIONS)
+else()
+  set(RunCMake_GENERATOR_INSTANCE "")
+  run_cmake(NoInstance)
 
-set(RunCMake_TEST_OPTIONS -DCMAKE_TOOLCHAIN_FILE=${RunCMake_SOURCE_DIR}/BadInstance-toolchain.cmake)
-run_cmake(BadInstanceToolchain)
-unset(RunCMake_TEST_OPTIONS)
+  set(RunCMake_GENERATOR_INSTANCE "Bad Instance")
+  run_cmake(BadInstance)
+
+  set(RunCMake_TEST_OPTIONS -DCMAKE_TOOLCHAIN_FILE=${RunCMake_SOURCE_DIR}/BadInstance-toolchain.cmake)
+  run_cmake(BadInstanceToolchain)
+  unset(RunCMake_TEST_OPTIONS)
+endif()