Merge topic 'cmake-server-filewatcher'

4e34f042 server-mode: Watch CMakeLists.txt files
26250002 server-mode: Report watched files to client
0d96e193 server-mode: Add infrastructure to watch the filesystem

Help/manual/cmake-server.7.rst

@@ -194,6 +194,49 @@ are of type "signal", have an empty "cookie" and "inReplyTo" field and always
 have a "name" set to show which signal was sent.
 
 
+Specific Signals
+----------------
+
+The cmake server may sent signals with the following names:
+
+"dirty" Signal
+^^^^^^^^^^^^^^
+
+The "dirty" signal is sent whenever the server determines that the configuration
+of the project is no longer up-to-date. This happens when any of the files that have
+an influence on the build system is changed.
+
+The "dirty" signal may look like this::
+
+  [== CMake Server ==[
+  {
+    "cookie":"",
+    "inReplyTo":"",
+    "name":"dirty",
+    "type":"signal"}
+  ]== CMake Server ==]
+
+
+"fileChange" Signal
+^^^^^^^^^^^^^^^^^^^
+
+The "fileChange" signal is sent whenever a watched file is changed. It contains
+the "path" that has changed and a list of "properties" with the kind of change
+that was detected. Possible changes are "change" and "rename".
+
+The "fileChange" signal looks like this::
+
+  [== CMake Server ==[
+  {
+    "cookie":"",
+    "inReplyTo":"",
+    "name":"fileChange",
+    "path":"/absolute/CMakeLists.txt",
+    "properties":["change"],
+    "type":"signal"}
+  ]== CMake Server ==]
+
+
 Specific Message Types
 ----------------------
 
@@ -635,3 +678,26 @@ CMake will respond with the following output::
 
 The output can be limited to a list of keys by passing an array of key names
 to the "keys" optional field of the "cache" request.
+
+
+Type "fileSystemWatchers"
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The server can watch the filesystem for changes. The "fileSystemWatchers"
+command will report on the files and directories watched.
+
+Example::
+
+  [== CMake Server ==]
+  {"type":"fileSystemWatchers"}
+  [== CMake Server ==]
+
+CMake will respond with the following output::
+
+  [== CMake Server ==]
+  {
+    "cookie":"","inReplyTo":"fileSystemWatchers","type":"reply",
+    "watchedFiles": [ "/absolute/path" ],
+    "watchedDirectories": [ "/absolute" ]
+  }
+  [== CMake Server ==]

Source/CMakeLists.txt

@@ -780,6 +780,7 @@ target_link_libraries(cmake CMakeLib)
 
 if(CMake_ENABLE_SERVER_MODE)
   add_library(CMakeServerLib
+    cmFileMonitor.cxx cmFileMonitor.h
     cmServer.cxx cmServer.h
     cmServerConnection.cxx cmServerConnection.h
     cmServerProtocol.cxx cmServerProtocol.h

Source/cmFileMonitor.cxx

@@ -0,0 +1,389 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+#include "cmFileMonitor.h"
+
+#include <cmsys/SystemTools.hxx>
+
+#include <cassert>
+#include <iostream>
+#include <set>
+#include <unordered_map>
+
+namespace {
+void on_directory_change(uv_fs_event_t* handle, const char* filename,
+                         int events, int status);
+void on_handle_close(uv_handle_t* handle);
+} // namespace
+
+class cmIBaseWatcher
+{
+public:
+  cmIBaseWatcher() = default;
+  virtual ~cmIBaseWatcher() = default;
+
+  virtual void Trigger(const std::string& pathSegment, int events,
+                       int status) const = 0;
+  virtual std::string Path() const = 0;
+  virtual uv_loop_t* Loop() const = 0;
+
+  virtual void StartWatching() = 0;
+  virtual void StopWatching() = 0;
+
+  virtual std::vector<std::string> WatchedFiles() const = 0;
+  virtual std::vector<std::string> WatchedDirectories() const = 0;
+};
+
+class cmVirtualDirectoryWatcher : public cmIBaseWatcher
+{
+public:
+  ~cmVirtualDirectoryWatcher()
+  {
+    for (auto i : this->Children) {
+      delete i.second;
+    }
+  }
+
+  cmIBaseWatcher* Find(const std::string& ps)
+  {
+    const auto i = this->Children.find(ps);
+    return (i == this->Children.end()) ? nullptr : i->second;
+  }
+
+  void Trigger(const std::string& pathSegment, int events,
+               int status) const final
+  {
+    if (pathSegment.empty()) {
+      for (const auto& i : this->Children) {
+        i.second->Trigger(std::string(), events, status);
+      }
+    } else {
+      const auto i = this->Children.find(pathSegment);
+      if (i != this->Children.end()) {
+        i->second->Trigger(std::string(), events, status);
+      }
+    }
+  }
+
+  void StartWatching() override
+  {
+    for (const auto& i : this->Children) {
+      i.second->StartWatching();
+    }
+  }
+
+  void StopWatching() override
+  {
+    for (const auto& i : this->Children) {
+      i.second->StopWatching();
+    }
+  }
+
+  std::vector<std::string> WatchedFiles() const final
+  {
+    std::vector<std::string> result;
+    for (const auto& i : this->Children) {
+      for (const auto& j : i.second->WatchedFiles()) {
+        result.push_back(j);
+      }
+    }
+    return result;
+  }
+
+  std::vector<std::string> WatchedDirectories() const override
+  {
+    std::vector<std::string> result;
+    for (const auto& i : this->Children) {
+      for (const auto& j : i.second->WatchedDirectories()) {
+        result.push_back(j);
+      }
+    }
+    return result;
+  }
+
+  void Reset()
+  {
+    for (auto c : this->Children) {
+      delete c.second;
+    }
+    this->Children.clear();
+  }
+
+  void AddChildWatcher(const std::string& ps, cmIBaseWatcher* watcher)
+  {
+    assert(!ps.empty());
+    assert(this->Children.find(ps) == this->Children.end());
+    assert(watcher);
+
+    this->Children.emplace(std::make_pair(ps, watcher));
+  }
+
+private:
+  std::unordered_map<std::string, cmIBaseWatcher*> Children; // owned!
+};
+
+// Root of all the different (on windows!) root directories:
+class cmRootWatcher : public cmVirtualDirectoryWatcher
+{
+public:
+  cmRootWatcher(uv_loop_t* loop)
+    : mLoop(loop)
+  {
+    assert(loop);
+  }
+
+  std::string Path() const final
+  {
+    assert(false);
+    return std::string();
+  }
+  uv_loop_t* Loop() const final { return this->mLoop; }
+
+private:
+  uv_loop_t* const mLoop; // no ownership!
+};
+
+// Real directories:
+class cmRealDirectoryWatcher : public cmVirtualDirectoryWatcher
+{
+public:
+  cmRealDirectoryWatcher(cmVirtualDirectoryWatcher* p, const std::string& ps)
+    : Parent(p)
+    , PathSegment(ps)
+  {
+    assert(p);
+    assert(!ps.empty());
+
+    p->AddChildWatcher(ps, this);
+  }
+
+  ~cmRealDirectoryWatcher()
+  {
+    // Handle is freed via uv_handle_close callback!
+  }
+
+  void StartWatching() final
+  {
+    if (!this->Handle) {
+      this->Handle = new uv_fs_event_t;
+
+      uv_fs_event_init(this->Loop(), this->Handle);
+      this->Handle->data = this;
+      uv_fs_event_start(this->Handle, &on_directory_change, Path().c_str(), 0);
+    }
+    cmVirtualDirectoryWatcher::StartWatching();
+  }
+
+  void StopWatching() final
+  {
+    if (this->Handle) {
+      uv_fs_event_stop(this->Handle);
+      uv_close(reinterpret_cast<uv_handle_t*>(this->Handle), &on_handle_close);
+      this->Handle = nullptr;
+    }
+    cmVirtualDirectoryWatcher::StopWatching();
+  }
+
+  uv_loop_t* Loop() const final { return this->Parent->Loop(); }
+
+  std::vector<std::string> WatchedDirectories() const override
+  {
+    std::vector<std::string> result = { Path() };
+    for (const auto& j : cmVirtualDirectoryWatcher::WatchedDirectories()) {
+      result.push_back(j);
+    }
+    return result;
+  }
+
+protected:
+  cmVirtualDirectoryWatcher* const Parent;
+  const std::string PathSegment;
+
+private:
+  uv_fs_event_t* Handle = nullptr; // owner!
+};
+
+// Root directories:
+class cmRootDirectoryWatcher : public cmRealDirectoryWatcher
+{
+public:
+  cmRootDirectoryWatcher(cmRootWatcher* p, const std::string& ps)
+    : cmRealDirectoryWatcher(p, ps)
+  {
+  }
+
+  std::string Path() const final { return this->PathSegment; }
+};
+
+// Normal directories below root:
+class cmDirectoryWatcher : public cmRealDirectoryWatcher
+{
+public:
+  cmDirectoryWatcher(cmRealDirectoryWatcher* p, const std::string& ps)
+    : cmRealDirectoryWatcher(p, ps)
+  {
+  }
+
+  std::string Path() const final
+  {
+    return this->Parent->Path() + this->PathSegment + "/";
+  }
+};
+
+class cmFileWatcher : public cmIBaseWatcher
+{
+public:
+  cmFileWatcher(cmRealDirectoryWatcher* p, const std::string& ps,
+                cmFileMonitor::Callback cb)
+    : Parent(p)
+    , PathSegment(ps)
+    , CbList({ cb })
+  {
+    assert(p);
+    assert(!ps.empty());
+    p->AddChildWatcher(ps, this);
+  }
+
+  void StartWatching() final {}
+
+  void StopWatching() final {}
+
+  void AppendCallback(cmFileMonitor::Callback cb) { CbList.push_back(cb); }
+
+  std::string Path() const final
+  {
+    return this->Parent->Path() + this->PathSegment;
+  }
+
+  std::vector<std::string> WatchedDirectories() const final { return {}; }
+
+  std::vector<std::string> WatchedFiles() const final
+  {
+    return { this->Path() };
+  }
+
+  void Trigger(const std::string& ps, int events, int status) const final
+  {
+    assert(ps.empty());
+    assert(status == 0);
+    static_cast<void>(ps);
+
+    const std::string path = this->Path();
+    for (const auto& cb : this->CbList) {
+      cb(path, events, status);
+    }
+  }
+
+  uv_loop_t* Loop() const final { return this->Parent->Loop(); }
+
+private:
+  cmRealDirectoryWatcher* Parent;
+  const std::string PathSegment;
+  std::vector<cmFileMonitor::Callback> CbList;
+};
+
+namespace {
+
+void on_directory_change(uv_fs_event_t* handle, const char* filename,
+                         int events, int status)
+{
+  const cmIBaseWatcher* const watcher =
+    static_cast<const cmIBaseWatcher*>(handle->data);
+  const std::string pathSegment(filename);
+  watcher->Trigger(pathSegment, events, status);
+}
+
+void on_handle_close(uv_handle_t* handle)
+{
+  delete (reinterpret_cast<uv_fs_event_t*>(handle));
+}
+
+} // namespace
+
+cmFileMonitor::cmFileMonitor(uv_loop_t* l)
+  : Root(new cmRootWatcher(l))
+{
+}
+
+cmFileMonitor::~cmFileMonitor()
+{
+  delete this->Root;
+}
+
+void cmFileMonitor::MonitorPaths(const std::vector<std::string>& paths,
+                                 Callback cb)
+{
+  for (const auto& p : paths) {
+    std::vector<std::string> pathSegments;
+    cmsys::SystemTools::SplitPath(p, pathSegments, true);
+
+    const size_t segmentCount = pathSegments.size();
+    if (segmentCount < 2) { // Expect at least rootdir and filename
+      continue;
+    }
+    cmVirtualDirectoryWatcher* currentWatcher = this->Root;
+    for (size_t i = 0; i < segmentCount; ++i) {
+      assert(currentWatcher);
+
+      const bool fileSegment = (i == segmentCount - 1);
+      const bool rootSegment = (i == 0);
+      assert(
+        !(fileSegment &&
+          rootSegment)); // Can not be both filename and root part of the path!
+
+      const std::string& currentSegment = pathSegments[i];
+
+      cmIBaseWatcher* nextWatcher = currentWatcher->Find(currentSegment);
+      if (!nextWatcher) {
+        if (rootSegment) { // Root part
+          assert(currentWatcher == this->Root);
+          nextWatcher = new cmRootDirectoryWatcher(this->Root, currentSegment);
+          assert(currentWatcher->Find(currentSegment) == nextWatcher);
+        } else if (fileSegment) { // File part
+          assert(currentWatcher != this->Root);
+          nextWatcher = new cmFileWatcher(
+            dynamic_cast<cmRealDirectoryWatcher*>(currentWatcher),
+            currentSegment, cb);
+          assert(currentWatcher->Find(currentSegment) == nextWatcher);
+        } else { // Any normal directory in between
+          nextWatcher = new cmDirectoryWatcher(
+            dynamic_cast<cmRealDirectoryWatcher*>(currentWatcher),
+            currentSegment);
+          assert(currentWatcher->Find(currentSegment) == nextWatcher);
+        }
+      } else {
+        if (fileSegment) {
+          auto filePtr = dynamic_cast<cmFileWatcher*>(nextWatcher);
+          assert(filePtr);
+          filePtr->AppendCallback(cb);
+          continue;
+        }
+      }
+      currentWatcher = dynamic_cast<cmVirtualDirectoryWatcher*>(nextWatcher);
+    }
+  }
+  this->Root->StartWatching();
+}
+
+void cmFileMonitor::StopMonitoring()
+{
+  this->Root->StopWatching();
+  this->Root->Reset();
+}
+
+std::vector<std::string> cmFileMonitor::WatchedFiles() const
+{
+  std::vector<std::string> result;
+  if (this->Root) {
+    result = this->Root->WatchedFiles();
+  }
+  return result;
+}
+
+std::vector<std::string> cmFileMonitor::WatchedDirectories() const
+{
+  std::vector<std::string> result;
+  if (this->Root) {
+    result = this->Root->WatchedDirectories();
+  }
+  return result;
+}

Source/cmFileMonitor.h

@@ -0,0 +1,28 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+#pragma once
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "cm_uv.h"
+
+class cmRootWatcher;
+
+class cmFileMonitor
+{
+public:
+  cmFileMonitor(uv_loop_t* l);
+  ~cmFileMonitor();
+
+  using Callback = std::function<void(const std::string&, int, int)>;
+  void MonitorPaths(const std::vector<std::string>& paths, Callback cb);
+  void StopMonitoring();
+
+  std::vector<std::string> WatchedFiles() const;
+  std::vector<std::string> WatchedDirectories() const;
+
+private:
+  cmRootWatcher* Root;
+};

Source/cmServer.cxx

@@ -237,6 +237,11 @@ bool cmServer::Serve(std::string* errorMessage)
   return Connection->ProcessEvents(errorMessage);
 }
 
+cmFileMonitor* cmServer::FileMonitor() const
+{
+  return Connection->FileMonitor();
+}
+
 void cmServer::WriteJsonObject(const Json::Value& jsonValue,
                                const DebugInfo* debug) const
 {

Source/cmServer.h

@@ -13,6 +13,7 @@
 #include <string>
 #include <vector>
 
+class cmFileMonitor;
 class cmServerConnection;
 class cmServerProtocol;
 class cmServerRequest;
@@ -28,6 +29,8 @@ public:
 
   bool Serve(std::string* errorMessage);
 
+  cmFileMonitor* FileMonitor() const;
+
 private:
   void RegisterProtocol(cmServerProtocol* protocol);
 

Source/cmServerConnection.cxx

@@ -4,7 +4,8 @@
 
 #include "cmServerDictionary.h"
 
-#include <cmServer.h>
+#include "cmFileMonitor.h"
+#include "cmServer.h"
 
 #include <assert.h>
 
@@ -64,10 +65,16 @@ public:
     : Connection(connection)
   {
     Connection->mLoop = uv_default_loop();
+    if (Connection->mLoop) {
+      Connection->mFileMonitor = new cmFileMonitor(Connection->mLoop);
+    }
   }
 
   ~LoopGuard()
   {
+    if (Connection->mFileMonitor) {
+      delete Connection->mFileMonitor;
+    }
     uv_loop_close(Connection->mLoop);
     Connection->mLoop = nullptr;
   }

Source/cmServerConnection.h

@@ -10,6 +10,7 @@
 #endif
 
 class cmServer;
+class cmFileMonitor;
 class LoopGuard;
 
 class cmServerConnection
@@ -29,6 +30,8 @@ public:
 
   virtual void Connect(uv_stream_t* server) { (void)(server); }
 
+  cmFileMonitor* FileMonitor() const { return this->mFileMonitor; }
+
 protected:
   virtual bool DoSetup(std::string* errorMessage) = 0;
   virtual void TearDown() = 0;
@@ -46,6 +49,7 @@ protected:
 
 private:
   uv_loop_t* mLoop = nullptr;
+  cmFileMonitor* mFileMonitor = nullptr;
   cmServer* Server = nullptr;
 
   friend class LoopGuard;

Source/cmServerDictionary.h

@@ -6,12 +6,16 @@
 
 // Vocabulary:
 
+static const std::string kDIRTY_SIGNAL = "dirty";
+static const std::string kFILE_CHANGE_SIGNAL = "fileChange";
+
 static const std::string kCACHE_TYPE = "cache";
 static const std::string kCMAKE_INPUTS_TYPE = "cmakeInputs";
 static const std::string kCODE_MODEL_TYPE = "codemodel";
 static const std::string kCOMPUTE_TYPE = "compute";
 static const std::string kCONFIGURE_TYPE = "configure";
 static const std::string kERROR_TYPE = "error";
+static const std::string kFILESYSTEM_WATCHERS_TYPE = "fileSystemWatchers";
 static const std::string kGLOBAL_SETTINGS_TYPE = "globalSettings";
 static const std::string kHANDSHAKE_TYPE = "handshake";
 static const std::string kMESSAGE_TYPE = "message";
@@ -80,6 +84,11 @@ static const std::string kVALUE_KEY = "value";
 static const std::string kWARN_UNINITIALIZED_KEY = "warnUninitialized";
 static const std::string kWARN_UNUSED_CLI_KEY = "warnUnusedCli";
 static const std::string kWARN_UNUSED_KEY = "warnUnused";
+static const std::string kWATCHED_DIRECTORIES_KEY = "watchedDirectories";
+static const std::string kWATCHED_FILES_KEY = "watchedFiles";
 
 static const std::string kSTART_MAGIC = "[== CMake Server ==[";
 static const std::string kEND_MAGIC = "]== CMake Server ==]";
+
+static const std::string kRENAME_PROPERTY_VALUE = "rename";
+static const std::string kCHANGE_PROPERTY_VALUE = "change";

Source/cmServerProtocol.cxx

@@ -4,6 +4,7 @@
 
 #include "cmCacheManager.h"
 #include "cmExternalMakefileProjectGenerator.h"
+#include "cmFileMonitor.h"
 #include "cmGeneratorTarget.h"
 #include "cmGlobalGenerator.h"
 #include "cmListFileCache.h"
@@ -214,6 +215,11 @@ bool cmServerProtocol::Activate(cmServer* server,
   return result;
 }
 
+cmFileMonitor* cmServerProtocol::FileMonitor() const
+{
+  return this->m_Server ? this->m_Server->FileMonitor() : nullptr;
+}
+
 void cmServerProtocol::SendSignal(const std::string& name,
                                   const Json::Value& data) const
 {
@@ -365,6 +371,30 @@ bool cmServerProtocol1_0::DoActivate(const cmServerRequest& request,
   return true;
 }
 
+void cmServerProtocol1_0::HandleCMakeFileChanges(const std::string& path,
+                                                 int event, int status)
+{
+  assert(status == 0);
+  static_cast<void>(status);
+
+  if (!m_isDirty) {
+    m_isDirty = true;
+    SendSignal(kDIRTY_SIGNAL, Json::objectValue);
+  }
+  Json::Value obj = Json::objectValue;
+  obj[kPATH_KEY] = path;
+  Json::Value properties = Json::arrayValue;
+  if (event & UV_RENAME) {
+    properties.append(kRENAME_PROPERTY_VALUE);
+  }
+  if (event & UV_CHANGE) {
+    properties.append(kCHANGE_PROPERTY_VALUE);
+  }
+
+  obj[kPROPERTIES_KEY] = properties;
+  SendSignal(kFILE_CHANGE_SIGNAL, obj);
+}
+
 const cmServerResponse cmServerProtocol1_0::Process(
   const cmServerRequest& request)
 {
@@ -385,6 +415,9 @@ const cmServerResponse cmServerProtocol1_0::Process(
   if (request.Type == kCONFIGURE_TYPE) {
     return this->ProcessConfigure(request);
   }
+  if (request.Type == kFILESYSTEM_WATCHERS_TYPE) {
+    return this->ProcessFileSystemWatchers(request);
+  }
   if (request.Type == kGLOBAL_SETTINGS_TYPE) {
     return this->ProcessGlobalSettings(request);
   }
@@ -862,6 +895,8 @@ cmServerResponse cmServerProtocol1_0::ProcessConfigure(
     return request.ReportError("This instance is inactive.");
   }
 
+  FileMonitor()->StopMonitoring();
+
   // Make sure the types of cacheArguments matches (if given):
   std::vector<std::string> cacheArgs;
   bool cacheArgumentsError = false;
@@ -938,7 +973,17 @@ cmServerResponse cmServerProtocol1_0::ProcessConfigure(
   if (ret < 0) {
     return request.ReportError("Configuration failed.");
   }
+
+  std::vector<std::string> toWatchList;
+  getCMakeInputs(gg, std::string(), buildDir, nullptr, &toWatchList, nullptr);
+
+  FileMonitor()->MonitorPaths(toWatchList,
+                              [this](const std::string& p, int e, int s) {
+                                this->HandleCMakeFileChanges(p, e, s);
+                              });
+
   m_State = STATE_CONFIGURED;
+  m_isDirty = false;
   return request.Reply(Json::Value());
 }
 
@@ -1011,3 +1056,22 @@ cmServerResponse cmServerProtocol1_0::ProcessSetGlobalSettings(
 
   return request.Reply(Json::Value());
 }
+
+cmServerResponse cmServerProtocol1_0::ProcessFileSystemWatchers(
+  const cmServerRequest& request)
+{
+  const cmFileMonitor* const fm = FileMonitor();
+  Json::Value result = Json::objectValue;
+  Json::Value files = Json::arrayValue;
+  for (const auto& f : fm->WatchedFiles()) {
+    files.append(f);
+  }
+  Json::Value directories = Json::arrayValue;
+  for (const auto& d : fm->WatchedDirectories()) {
+    directories.append(d);
+  }
+  result[kWATCHED_FILES_KEY] = files;
+  result[kWATCHED_DIRECTORIES_KEY] = directories;
+
+  return request.Reply(result);
+}

Source/cmServerProtocol.h

@@ -13,6 +13,7 @@
 #include <string>
 
 class cmake;
+class cmFileMonitor;
 class cmServer;
 
 class cmServerRequest;
@@ -81,6 +82,7 @@ public:
   bool Activate(cmServer* server, const cmServerRequest& request,
                 std::string* errorMessage);
 
+  cmFileMonitor* FileMonitor() const;
   void SendSignal(const std::string& name, const Json::Value& data) const;
 
 protected:
@@ -107,6 +109,8 @@ private:
   bool DoActivate(const cmServerRequest& request,
                   std::string* errorMessage) override;
 
+  void HandleCMakeFileChanges(const std::string& path, int event, int status);
+
   // Handle requests:
   cmServerResponse ProcessCache(const cmServerRequest& request);
   cmServerResponse ProcessCMakeInputs(const cmServerRequest& request);
@@ -115,6 +119,7 @@ private:
   cmServerResponse ProcessConfigure(const cmServerRequest& request);
   cmServerResponse ProcessGlobalSettings(const cmServerRequest& request);
   cmServerResponse ProcessSetGlobalSettings(const cmServerRequest& request);
+  cmServerResponse ProcessFileSystemWatchers(const cmServerRequest& request);
 
   enum State
   {
@@ -124,4 +129,6 @@ private:
     STATE_COMPUTED
   };
   State m_State = STATE_INACTIVE;
+
+  bool m_isDirty = false;
 };