ENH: Improve performance of check build system by creating another file that is simpler to parse and therefore much faster overall

Source/cmDepends.cxx

@@ -18,28 +18,34 @@
 
 #include "cmGeneratedFileStream.h"
 #include "cmSystemTools.h"
+#include "cmFileTimeComparison.h"
 
 #include <assert.h>
 
 //----------------------------------------------------------------------------
-cmDepends::cmDepends()
+cmDepends::cmDepends(): m_Verbose(false), m_FileComparison(0),
+  m_MaxPath(cmSystemTools::GetMaximumFilePathLength()),
+  m_Dependee(new char[m_MaxPath]),
+  m_Depender(new char[m_MaxPath])
 {
-  m_Verbose = false;
 }
 
 //----------------------------------------------------------------------------
 cmDepends::~cmDepends()
 {
+  delete [] m_Dependee;
+  delete [] m_Depender;
 }
 
 //----------------------------------------------------------------------------
-bool cmDepends::Write(const char *src, const char *obj, std::ostream &fout)
+bool cmDepends::Write(const char *src, const char *obj,
+  std::ostream &makeDepends, std::ostream &internalDepends)
 {
-  return this->WriteDependencies(src, obj, fout);
+  return this->WriteDependencies(src, obj, makeDepends, internalDepends);
 }
 
 //----------------------------------------------------------------------------
-void cmDepends::Check(const char *file)
+void cmDepends::Check(const char *makeFile, const char *internalFile)
 {
   // Dependency checks must be done in proper working directory.
   std::string oldcwd = ".";
@@ -52,11 +58,12 @@ void cmDepends::Check(const char *file)
     }
 
   // Check whether dependencies must be regenerated.
-  std::ifstream fin(file);
+  std::ifstream fin(internalFile);
   if(!(fin && this->CheckDependencies(fin)))
     {
     // Clear all dependencies so they will be regenerated.
-    this->Clear(file);
+    this->Clear(makeFile);
+    this->Clear(internalFile);
     }
 
   // Restore working directory.
@@ -82,6 +89,7 @@ void cmDepends::Clear(const char *file)
   std::string markFile = file;
   markFile += ".mark";
   cmSystemTools::RemoveFile(markFile.c_str());
+  std::cout << "Remove mark file: " << markFile.c_str() << std::endl;
   
   // Write an empty dependency file.
   cmGeneratedFileStream depFileStream(file);
@@ -90,3 +98,90 @@ void cmDepends::Clear(const char *file)
     << "# This may be replaced when dependencies are built." << std::endl;
 }
 
+//----------------------------------------------------------------------------
+bool cmDepends::CheckDependencies(std::istream& internalDepends)
+{
+  // Parse dependencies from the stream.  If any dependee is missing
+  // or newer than the depender then dependencies should be
+  // regenerated.
+  bool okay = true;
+  while(internalDepends.getline(m_Dependee, m_MaxPath))
+    {
+    if ( m_Dependee[0] == 0 || m_Dependee[0] == '#' || m_Dependee[0] == '\r' )
+      {
+      continue;
+      }
+    size_t len = internalDepends.gcount()-1;
+    if ( m_Dependee[len-1] == '\r' )
+      {
+      len --;
+      m_Dependee[len] = 0;
+      }
+    if ( m_Dependee[0] != ' ' )
+      {
+      memcpy(m_Depender, m_Dependee, len+1);
+      continue;
+      }
+    /*
+    // Parse the dependency line.
+    if(!this->ParseDependency(line.c_str()))
+      {
+      continue;
+      }
+      */
+
+    // Dependencies must be regenerated if the dependee does not exist
+    // or if the depender exists and is older than the dependee.
+    bool regenerate = false;
+    const char* dependee = m_Dependee+1;
+    const char* depender = m_Depender;
+    if(!cmSystemTools::FileExists(dependee))
+      {
+      // The dependee does not exist.
+      regenerate = true;
+
+      // Print verbose output.
+      if(m_Verbose)
+        {
+        cmOStringStream msg;
+        msg << "Dependee \"" << dependee
+            << "\" does not exist for depender \""
+            << depender << "\"." << std::endl;
+        cmSystemTools::Stdout(msg.str().c_str());
+        }
+      }
+    else if(cmSystemTools::FileExists(depender))
+      {
+      // The dependee and depender both exist.  Compare file times.
+      int result = 0;
+      if((!m_FileComparison->FileTimeCompare(depender, dependee,
+                                             &result) || result < 0))
+        {
+        // The depender is older than the dependee.
+        regenerate = true;
+
+        // Print verbose output.
+        if(m_Verbose)
+          {
+          cmOStringStream msg;
+          msg << "Dependee \"" << dependee
+              << "\" is newer than depender \""
+              << depender << "\"." << std::endl;
+          cmSystemTools::Stdout(msg.str().c_str());
+          }
+        }
+      }
+    if(regenerate)
+      {
+      // Dependencies must be regenerated.
+      okay = false;
+
+      // Remove the depender to be sure it is rebuilt.
+      cmSystemTools::RemoveFile(depender);
+      }
+    }
+
+  return okay;
+}
+
+

Source/cmDepends.h

@@ -19,6 +19,8 @@
 
 #include "cmStandardIncludes.h"
 
+class cmFileTimeComparison;
+
 /** \class cmDepends
  * \brief Dependency scanner superclass.
  *
@@ -43,25 +45,29 @@ public:
   virtual ~cmDepends();
 
   /** Write dependencies for the target file.  */
-  bool Write(const char *src, const char *obj, std::ostream &os);
+  bool Write(const char *src, const char *obj,
+    std::ostream &makeDepends, std::ostream &internalDepends);
   
   /** Check dependencies for the target file.  */
-  void Check(const char *file);
+  void Check(const char *makeFile, const char* internalFile);
 
   /** Clear dependencies for the target file so they will be regenerated.  */
   void Clear(const char *file);
 
+  /** Set the file comparison object */
+  void SetFileComparison(cmFileTimeComparison* fc) { m_FileComparison = fc; }
+
 protected:
 
   // Write dependencies for the target file to the given stream.
   // Return true for success and false for failure.
-  virtual bool WriteDependencies(const char *src, 
-                                 const char* obj, std::ostream& os)=0;
+  virtual bool WriteDependencies(const char *src, const char* obj,
+    std::ostream& makeDepends, std::ostream& internalDepends)=0;
 
   // Check dependencies for the target file in the given stream.
   // Return false if dependencies must be regenerated and true
   // otherwise.
-  virtual bool CheckDependencies(std::istream& is) = 0;
+  virtual bool CheckDependencies(std::istream& internalDepends);
 
   // The directory in which the build rule for the target file is executed.
   std::string m_Directory;
@@ -69,6 +75,11 @@ protected:
 
   // Flag for verbose output.
   bool m_Verbose;
+  cmFileTimeComparison* m_FileComparison;
+
+  size_t m_MaxPath;
+  char* m_Dependee;
+  char* m_Depender;
 
 private:
   cmDepends(cmDepends const&); // Purposely not implemented.

Source/cmDependsC.cxx

@@ -45,8 +45,8 @@ cmDependsC::~cmDependsC()
 }
 
 //----------------------------------------------------------------------------
-bool cmDependsC::WriteDependencies(const char *src, 
-                                   const char *obj, std::ostream& os)
+bool cmDependsC::WriteDependencies(const char *src, const char *obj,
+  std::ostream& makeDepends, std::ostream& internalDepends)
 {
   // Make sure this is a scanning instance.
   if(!src || src[0] == '\0')
@@ -161,88 +161,20 @@ bool cmDependsC::WriteDependencies(const char *src,
     }
 
   // Write the dependencies to the output stream.
+  internalDepends << obj << std::endl;
   for(std::set<cmStdString>::iterator i=dependencies.begin();
       i != dependencies.end(); ++i)
     {
-    os << obj << ": "
+    makeDepends << obj << ": "
        << cmSystemTools::ConvertToOutputPath(i->c_str()).c_str()
        << std::endl;
+    internalDepends << " " << i->c_str() << std::endl;
     }
-  os << std::endl;
+  makeDepends << std::endl;
 
   return true;
 }
 
-//----------------------------------------------------------------------------
-bool cmDependsC::CheckDependencies(std::istream& is)
-{
-  // Parse dependencies from the stream.  If any dependee is missing
-  // or newer than the depender then dependencies should be
-  // regenerated.
-  bool okay = true;
-  std::string line;
-  std::string depender;
-  std::string dependee;
-  while(cmSystemTools::GetLineFromStream(is, line))
-    {
-    // Parse the dependency line.
-    if(!this->ParseDependency(line.c_str(), depender, dependee))
-      {
-      continue;
-      }
-
-    // Dependencies must be regenerated if the dependee does not exist
-    // or if the depender exists and is older than the dependee.
-    bool regenerate = false;
-    if(!cmSystemTools::FileExists(dependee.c_str()))
-      {
-      // The dependee does not exist.
-      regenerate = true;
-
-      // Print verbose output.
-      if(m_Verbose)
-        {
-        cmOStringStream msg;
-        msg << "Dependee \"" << dependee
-            << "\" does not exist for depender \""
-            << depender << "\"." << std::endl;
-        cmSystemTools::Stdout(msg.str().c_str());
-        }
-      }
-    else if(cmSystemTools::FileExists(depender.c_str()))
-      {
-      // The dependee and depender both exist.  Compare file times.
-      int result = 0;
-      if((!cmSystemTools::FileTimeCompare(depender.c_str(), dependee.c_str(),
-                                          &result) || result < 0))
-        {
-        // The depender is older than the dependee.
-        regenerate = true;
-
-        // Print verbose output.
-        if(m_Verbose)
-          {
-          cmOStringStream msg;
-          msg << "Dependee \"" << dependee
-              << "\" is newer than depender \""
-              << depender << "\"." << std::endl;
-          cmSystemTools::Stdout(msg.str().c_str());
-          }
-        }
-      }
-    if(regenerate)
-      {
-      // Dependencies must be regenerated.
-      okay = false;
-
-      // Remove the depender to be sure it is rebuilt.
-      cmSystemTools::RemoveFile(depender.c_str());
-      }
-    }
-
-  return okay;
-}
-
 //----------------------------------------------------------------------------
 void cmDependsC::Scan(std::istream& is, const char* directory)
 {
@@ -282,76 +214,6 @@ void cmDependsC::Scan(std::istream& is, const char* directory)
     }
 }
 
-//----------------------------------------------------------------------------
-bool cmDependsC::ParseDependency(const char* line, std::string& depender,
-                                 std::string& dependee)
-{
-  // Start with empty names.
-  depender = "";
-  dependee = "";
-
-  // Get the left-hand-side of the dependency.
-  const char* c = this->ParseFileName(line, depender);
-
-  // Skip the ':' separator.
-  for(;c && *c && isspace(*c);++c);
-  if(!c || !*c || *c != ':')
-    {
-    return false;
-    }
-  ++c;
-
-  // Get the right-hand-side of the dependency.
-  return this->ParseFileName(c, dependee)?true:false;
-}
-
-//----------------------------------------------------------------------------
-const char* cmDependsC::ParseFileName(const char* in, std::string& name)
-{
-  // Skip leading whitespace.
-  const char* c = in;
-  for(;c && *c && isspace(*c);++c);
-
-  // If this is an empty line or a comment line return failure.
-  if(!c || !*c || *c == '#')
-    {
-    return 0;
-    }
-
-  // Parse the possibly quoted file name.
-  bool quoted = false;
-  char* buf = new char[strlen(in)+1];
-  char* pos = buf;
-  
-  // for every character while we haven't hit the end of the string AND we
-  // are in a quoted string OR the current character isn't a : or the second
-  // character AND it isn't a space
-  for(;*c && (quoted ||
-              ((*c != ':' || pos == buf+1) && !isspace(*c))); ++c)
-    {
-    if(*c == '"')
-      {
-      quoted = !quoted;
-      }
-    // handle unquoted escaped spaces
-    else if(!quoted && *c == '\\' && isspace(*(c+1)))
-      {
-      *pos =  *(++c);
-      pos++;
-      }
-    else
-      {
-      *pos = *c;
-      pos++;
-      }
-    }
-  *pos =0;
-  name += buf;
-  delete [] buf;
-  // Return the ending position.
-  return c;
-}
-
 //----------------------------------------------------------------------------
 bool cmDependsC::FileExistsOrIsGenerated(const std::string& fname,
                                          std::set<cmStdString>& scanned,

Source/cmDependsC.h

@@ -38,19 +38,17 @@ public:
   virtual ~cmDependsC();
   
 protected:
+  typedef std::vector<char> t_CharBuffer;
+
   // Implement writing/checking methods required by superclass.
   virtual bool WriteDependencies(const char *src, 
-                                 const char *file, std::ostream& os);
-  virtual bool CheckDependencies(std::istream& is);
+                                 const char *file,
+                                 std::ostream& makeDepends,
+                                 std::ostream& internalDepends);
 
   // Method to scan a single file.
   void Scan(std::istream& is, const char* directory);
 
-  // Method to parse a single dependency line.
-  bool ParseDependency(const char* line, std::string& depender,
-                       std::string& dependee);
-  const char* ParseFileName(const char* in, std::string& name);
-
   // Method to test for the existence of a file.
   bool FileExistsOrIsGenerated(const std::string& fname,
                                std::set<cmStdString>& scanned,
@@ -78,6 +76,7 @@ protected:
   };
   std::set<cmStdString> m_Encountered;
   std::queue<UnscannedEntry> m_Unscanned;
+  t_CharBuffer m_Buffer;
 
 private:
   cmDependsC(cmDependsC const&); // Purposely not implemented.

Source/cmDependsFortran.cxx

@@ -93,8 +93,8 @@ cmDependsFortran::~cmDependsFortran()
 }
 
 //----------------------------------------------------------------------------
-bool cmDependsFortran::WriteDependencies(const char *src,
-                                         const char *obj, std::ostream& os)
+bool cmDependsFortran::WriteDependencies(const char *src, const char *obj,
+  std::ostream& makeDepends, std::ostream& internalDepends)
 {
   // Make sure this is a scanning instance.
   if(!src || src[0] == '\0')
@@ -127,16 +127,19 @@ bool cmDependsFortran::WriteDependencies(const char *src,
     }
 
   // Write the include dependencies to the output stream.
+  internalDepends << obj << std::endl;
   for(std::set<cmStdString>::const_iterator i = parser.Includes.begin();
       i != parser.Includes.end(); ++i)
     {
-    os << obj << ": "
+    makeDepends << obj << ": "
        << cmSystemTools::ConvertToOutputPath(i->c_str()).c_str()
        << std::endl;
+    internalDepends << " " << i->c_str() << std::endl;
     }
-  os << std::endl;
+  makeDepends << std::endl;
 
   // Write module requirements to the output stream.
+  internalDepends << obj << ".requires" << std::endl;
   for(std::set<cmStdString>::const_iterator i = parser.Requires.begin();
       i != parser.Requires.end(); ++i)
     {
@@ -144,23 +147,26 @@ bool cmDependsFortran::WriteDependencies(const char *src,
     if(parser.Provides.find(*i) == parser.Provides.end())
       {
       // since we require some things add them to our list of requirements
-      os << obj << ".requires: " << i->c_str() << ".mod.proxy"
+      makeDepends << obj << ".requires: " << i->c_str() << ".mod.proxy"
          << std::endl;
+      internalDepends << " " << i->c_str() << ".mod.proxy" << std::endl;
       }
     }
 
   // Write provided modules to the output stream.
+  internalDepends << obj << ".mod.proxy" << std::endl;
   for(std::set<cmStdString>::const_iterator i = parser.Provides.begin();
       i != parser.Provides.end(); ++i)
     {
-    os << i->c_str() << ".mod.proxy: " << obj
-       << ".provides" << std::endl;
+    makeDepends << i->c_str() << ".mod.proxy: " << obj
+      << ".provides" << std::endl;
+    internalDepends << " " << i->c_str() << ".provides" << std::endl;
     }
   
   // If any modules are provided then they must be converted to stamp files.
   if(!parser.Provides.empty())
     {
-    os << obj << ".provides.build:\n";
+    makeDepends << obj << ".provides.build:\n";
     for(std::set<cmStdString>::const_iterator i = parser.Provides.begin();
         i != parser.Provides.end(); ++i)
       {
@@ -168,10 +174,10 @@ bool cmDependsFortran::WriteDependencies(const char *src,
       // cmake_copy_f90_mod will call back to this class, which will
       // try various cases for the real mod file name.
       std::string m = cmSystemTools::LowerCase(*i);
-      os << "\t$(CMAKE_COMMAND) -E cmake_copy_f90_mod "
+      makeDepends << "\t$(CMAKE_COMMAND) -E cmake_copy_f90_mod "
          << i->c_str() << " " << m.c_str() << ".mod.stamp\n";
       }
-    os << "\t@touch " << obj << ".provides.build\n";
+    makeDepends << "\t@touch " << obj << ".provides.build\n";
     }
 
   /*
@@ -231,13 +237,6 @@ bool cmDependsFortran::WriteDependencies(const char *src,
   return true;
 }
 
-//----------------------------------------------------------------------------
-bool cmDependsFortran::CheckDependencies(std::istream&)
-{
-  // TODO: Parse and check dependencies.
-  return true;
-}
-
 //----------------------------------------------------------------------------
 bool cmDependsFortran::CopyModule(const std::vector<std::string>& args)
 {

Source/cmDependsFortran.h

@@ -51,9 +51,9 @@ public:
 
 protected:
   // Implement writing/checking methods required by superclass.
-  virtual bool WriteDependencies(const char *src,
-                                 const char *file, std::ostream& os);
-  virtual bool CheckDependencies(std::istream& is);
+  virtual bool WriteDependencies(
+    const char *src, const char *file,
+    std::ostream& makeDepends, std::ostream& internalDepends);
 
   // The source file from which to start scanning.
   std::string m_SourceFile;

Source/cmDependsJava.cxx

@@ -30,8 +30,8 @@ cmDependsJava::~cmDependsJava()
 }
 
 //----------------------------------------------------------------------------
-bool cmDependsJava::WriteDependencies(const char *src, 
-                                      const char *, std::ostream&)
+bool cmDependsJava::WriteDependencies(const char *src, const char *,
+  std::ostream&, std::ostream&)
 {
   // Make sure this is a scanning instance.
   if(!src || src[0] == '\0')
@@ -43,7 +43,6 @@ bool cmDependsJava::WriteDependencies(const char *src,
   return true;
 }
 
-//----------------------------------------------------------------------------
 bool cmDependsJava::CheckDependencies(std::istream&)
 {
   return true;

Source/cmDependsJava.h

@@ -34,9 +34,9 @@ public:
 
 protected:
   // Implement writing/checking methods required by superclass.
-  virtual bool WriteDependencies(const char *src,
-                                 const char *file, std::ostream& os);
-  virtual bool CheckDependencies(std::istream& is);
+  virtual bool WriteDependencies(const char *src, const char *file,
+    std::ostream& makeDepends, std::ostream& internalDepends);
+  virtual bool CheckDependencies(std::istream& internalDepends);
 
 private:
   cmDependsJava(cmDependsJava const&); // Purposely not implemented.

Source/cmLocalUnixMakefileGenerator3.cxx

@@ -826,7 +826,7 @@ cmLocalUnixMakefileGenerator3
     // Check the dependencies. Ths is required because we need at least an
     // empty depends.make for make to include, so at cmake time the
     // ::Check() method will generate that if it does not exist
-    checker->Check(objFile);
+    checker->Check(objFile, 0);
     
     return true;
     }
@@ -2673,6 +2673,7 @@ cmLocalUnixMakefileGenerator3::GetDependsChecker(const std::string& lang,
   if (ret)
     {
     ret->SetVerbose(verbose);
+    ret->SetFileComparison(m_GlobalGenerator->GetCMakeInstance()->GetFileComparison());
     }
   return ret;
 }
@@ -2734,18 +2735,28 @@ cmLocalUnixMakefileGenerator3
 
   // create the file stream for the depends file
   std::string dir = cmSystemTools::GetFilenamePath(infoFile);
-  dir += "/depend.make";
   
   // Open the rule file.  This should be copy-if-different because the
   // rules may depend on this file itself.
   std::string ruleFileNameFull = dir;
+  ruleFileNameFull += "/depend.make";
   cmGeneratedFileStream ruleFileStream(ruleFileNameFull.c_str());
   ruleFileStream.SetCopyIfDifferent(true);
   if(!ruleFileStream)
     {
     return false;
     }
+  std::string internalRuleFileNameFull = dir;
+  internalRuleFileNameFull += "/depend.internal";
+  cmGeneratedFileStream internalRuleFileStream(internalRuleFileNameFull.c_str());
+  internalRuleFileStream.SetCopyIfDifferent(true);
+  if(!internalRuleFileStream)
+    {
+    return false;
+    }
+
   this->WriteDisclaimer(ruleFileStream);
+  this->WriteDisclaimer(internalRuleFileStream);
 
   // Get the set of generated files.
   std::vector<std::string> generatedFilesVec;
@@ -2832,6 +2843,7 @@ cmLocalUnixMakefileGenerator3
     
     if (scanner)
       {
+      scanner->SetFileComparison(m_GlobalGenerator->GetCMakeInstance()->GetFileComparison());
       // for each file we need to scan
       std::string srcLang = "CMAKE_DEPENDS_CHECK_";
       srcLang += lang;
@@ -2846,7 +2858,7 @@ cmLocalUnixMakefileGenerator3
         // make sure the object file is relative to home output
         std::string obj = *si;
         obj = lg->Convert(obj.c_str(),HOME_OUTPUT,MAKEFILE);
-        scanner->Write(src.c_str(),obj.c_str(),ruleFileStream);
+        scanner->Write(src.c_str(),obj.c_str(),ruleFileStream, internalRuleFileStream);
         }
 
       // free the scanner for this language
@@ -2855,8 +2867,8 @@ cmLocalUnixMakefileGenerator3
     }
 
   // dependencies were generated, so touch the mark file
-  dir += ".mark";
-  std::ofstream fmark(dir.c_str());
+  ruleFileNameFull += ".mark";
+  std::ofstream fmark(ruleFileNameFull.c_str());
   fmark << "Dependencies updated>" << std::endl;
   
   return true;
@@ -3043,19 +3055,22 @@ void cmLocalUnixMakefileGenerator3::CheckDependencies(cmMakefile* mf,
   // For each info file run the check
   cmDependsC checker;
   checker.SetVerbose(verbose);
+  checker.SetFileComparison(m_GlobalGenerator->GetCMakeInstance()->GetFileComparison());
   for(std::vector<std::string>::iterator l = files.begin();
       l != files.end(); ++l)
     {
     // either clear or check the files
-    std::string dependFile = cmSystemTools::GetFilenamePath(l->c_str());
-    dependFile += "/depend.make";
+    std::string dir = cmSystemTools::GetFilenamePath(l->c_str());
+    std::string internalDependFile = dir + "/depend.internal";
+    std::string dependFile = dir + "/depend.make";
     if (clear)
       {
+      checker.Clear(internalDependFile.c_str());
       checker.Clear(dependFile.c_str());
       }
     else
       {
-      checker.Check(dependFile.c_str());
+      checker.Check(dependFile.c_str(), internalDependFile.c_str());
       }
     }
 }