cmState: Avoid accumulating snapshot storage for backtraces

Changes during post-3.3/pre-3.4 development refactored storage of most
configure-time information, including variable bindings and function
scopes.  All scopes (even short-lived) were kept persistently for
possible future debugging features, causing huge accumulated memory
usage.  This was mostly addressed by commit v3.4.1~4^2 (cmState: Avoid
accumulating snapshot storage for short-lived scopes, 2015-11-24).

Since then we still keep short-lived scopes when they are needed for a
backtrace.  This is because since commit v3.4.0-rc1~378^2
(cmListFileBacktrace: Implement in terms of cmState::Snapshot,
2015-05-29) backtraces have been lightweight objects that simply point
into the snapshot tree.  While the intention of this approach was to
avoid duplicating the call stack file path strings, the cost turned out
to be holding on to the entire call stack worth of scope snapshots,
which is much worse.

Furthermore, since commit v3.4.0-rc2~1^2 (cmIfCommand: Issue CMP0054
warning with appropriate context, 2015-10-20) all conditions used in
`if()` commands hold a backtrace for use in diagnostic messages.  Even
though the backtrace is short-lived it still causes the scope snapshot
to be kept.  This means that code like

    function(foo)
      if(0)
      endif()
    endfunction()

    foreach(i RANGE 1000000)
      foo()
    endforeach()

accumulates storage for the function call scope snapshots.

Fix this by partially reverting commit v3.4.0-rc1~378^2 and saving the
entire call stack during cmListFileBacktrace construction.  This way
we can avoid keeping short-lived scope snapshot storage in all cases.

Source/cmListFileCache.cxx

@@ -400,13 +400,40 @@ bool cmListFileParser::AddArgument(cmListFileLexer_Token* token,
 
 cmListFileBacktrace::cmListFileBacktrace(cmState::Snapshot snapshot,
                                          cmCommandContext const& cc)
-  : Context(cc)
-  , Snapshot(snapshot)
+  : Snapshot(snapshot)
 {
-  if (this->Snapshot.IsValid())
+  if (!this->Snapshot.IsValid())
+    {
+    return;
+    }
+
+  // Record the entire call stack now so that the `Snapshot` we
+  // save for later refers to a long-lived scope.  This avoids
+  // having to keep short-lived scopes around just to extract
+  // their backtrace information later.
+
+  cmListFileContext lfc =
+    cmListFileContext::FromCommandContext(
+      cc, this->Snapshot.GetExecutionListFile());
+  this->push_back(lfc);
+
+  cmState::Snapshot parent = this->Snapshot.GetCallStackParent();
+  while (parent.IsValid())
     {
-    this->Snapshot.Keep();
+    lfc.Name = this->Snapshot.GetEntryPointCommand();
+    lfc.Line = this->Snapshot.GetEntryPointLine();
+    lfc.FilePath = parent.GetExecutionListFile();
+    if (lfc.FilePath.empty())
+      {
+      break;
+      }
+    this->push_back(lfc);
+
+    this->Snapshot = parent;
+    parent = parent.GetCallStackParent();
     }
+
+  this->Snapshot = this->Snapshot.GetCallStackBottom();
 }
 
 cmListFileBacktrace::~cmListFileBacktrace()
@@ -415,48 +442,30 @@ cmListFileBacktrace::~cmListFileBacktrace()
 
 void cmListFileBacktrace::PrintTitle(std::ostream& out) const
 {
-  if (!this->Snapshot.IsValid())
+  if (this->empty())
     {
     return;
     }
   cmOutputConverter converter(this->Snapshot);
-  cmListFileContext lfc =
-      cmListFileContext::FromCommandContext(
-        this->Context, this->Snapshot.GetExecutionListFile());
+  cmListFileContext lfc = this->front();
   lfc.FilePath = converter.Convert(lfc.FilePath, cmOutputConverter::HOME);
   out << (lfc.Line ? " at " : " in ") << lfc;
 }
 
 void cmListFileBacktrace::PrintCallStack(std::ostream& out) const
 {
-  if (!this->Snapshot.IsValid())
-    {
-    return;
-    }
-  cmState::Snapshot parent = this->Snapshot.GetCallStackParent();
-  if (!parent.IsValid() || parent.GetExecutionListFile().empty())
+  if (this->size() <= 1)
     {
     return;
     }
+  out << "Call Stack (most recent call first):\n";
 
   cmOutputConverter converter(this->Snapshot);
-  std::string commandName = this->Snapshot.GetEntryPointCommand();
-  long commandLine = this->Snapshot.GetEntryPointLine();
-
-  out << "Call Stack (most recent call first):\n";
-  while(parent.IsValid())
+  for (const_iterator i = this->begin() + 1; i != this->end(); ++i)
     {
-    cmListFileContext lfc;
-    lfc.Name = commandName;
-    lfc.Line = commandLine;
-
-    lfc.FilePath = converter.Convert(parent.GetExecutionListFile(),
-                                     cmOutputConverter::HOME);
+    cmListFileContext lfc = *i;
+    lfc.FilePath = converter.Convert(lfc.FilePath, cmOutputConverter::HOME);
     out << "  " << lfc << "\n";
-
-    commandName = parent.GetEntryPointCommand();
-    commandLine = parent.GetEntryPointLine();
-    parent = parent.GetCallStackParent();
     }
 }
 

Source/cmListFileCache.h

@@ -87,7 +87,7 @@ struct cmListFileFunction: public cmCommandContext
   std::vector<cmListFileArgument> Arguments;
 };
 
-class cmListFileBacktrace
+class cmListFileBacktrace: private std::vector<cmListFileContext>
 {
   public:
     cmListFileBacktrace(cmState::Snapshot snapshot = cmState::Snapshot(),
@@ -97,7 +97,6 @@ class cmListFileBacktrace
     void PrintTitle(std::ostream& out) const;
     void PrintCallStack(std::ostream& out) const;
   private:
-    cmCommandContext Context;
     cmState::Snapshot Snapshot;
 };
 

Source/cmState.cxx

@@ -1098,11 +1098,6 @@ void cmState::Directory::SetCurrentBinary(std::string const& dir)
   this->Snapshot_.SetDefinition("CMAKE_CURRENT_BINARY_DIR", loc.c_str());
 }
 
-void cmState::Snapshot::Keep()
-{
-  this->Position->Keep = true;
-}
-
 void cmState::Snapshot::SetListFile(const std::string& listfile)
 {
   *this->Position->ExecutionListFile = listfile;

Source/cmState.h

@@ -63,7 +63,6 @@ public:
     std::vector<std::string> ClosureKeys() const;
     bool RaiseScope(std::string const& var, const char* varDef);
 
-    void Keep();
     void SetListFile(std::string const& listfile);
 
     std::string GetExecutionListFile() const;