Rewrite CTest child output handling

This commit fixes cmCTestRunTest and cmProcess to more efficiently
handle child output.  We now use the buffer for each child output pipe
to hold at most a partial line plus one new block of data at a time.
All complete lines are scanned in-place, and then only the partial line
at the end of the buffer is moved back to the beginning before appending
new data.

We also simplify the cmProcess interface by making GetNextOutputLine the
only method that needs to be called while the process is running.  This
simplifies cmCTestRunTest so that CheckOutput can be called until it
returns false when the process is done.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -226,9 +226,7 @@ bool cmCTestMultiProcessHandler::CheckOutput()
       i != this->RunningTests.end(); ++i)
     {
     cmCTestRunTest* p = *i;
-    p->CheckOutput(); //reads and stores the process output
-    
-    if(!p->IsRunning())
+    if(!p->CheckOutput())
       {
       finished.push_back(p);
       }

Source/CTest/cmCTestRunTest.cxx

@@ -30,57 +30,36 @@ cmCTestRunTest::~cmCTestRunTest()
 {
 }
 
-bool cmCTestRunTest::IsRunning()
+//----------------------------------------------------------------------------
+bool cmCTestRunTest::CheckOutput()
 {
-  return this->TestProcess->IsRunning();
-}
-
-//---------------------------------------------------------
-//waits .1 sec for output from this process.
-void cmCTestRunTest::CheckOutput()
-{
-  std::string out, err;
-  bool running = this->TestProcess->CheckOutput(.1);
-  //start our timeout for reading the process output
-  double clock_start = cmSystemTools::GetTime();
-  int pipe;
-  bool gotStdOut = false;
-  bool gotStdErr = false;
-  while((pipe = this->TestProcess->
-        GetNextOutputLine(out, err, gotStdOut, gotStdErr, running) )
-        != cmsysProcess_Pipe_Timeout)
+  // Read lines for up to 0.1 seconds of total time.
+  double timeout = 0.1;
+  double timeEnd = cmSystemTools::GetTime() + timeout;
+  std::string line;
+  while((timeout = timeEnd - cmSystemTools::GetTime(), timeout > 0))
     {
-    if(pipe == cmsysProcess_Pipe_STDOUT ||
-       pipe == cmsysProcess_Pipe_STDERR ||
-       pipe == cmsysProcess_Pipe_None)
+    int p = this->TestProcess->GetNextOutputLine(line, timeout);
+    if(p == cmsysProcess_Pipe_None)
       {
-      if(gotStdErr)
-        {
-        cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
-                   this->GetIndex() << ": " << err << std::endl);
-        this->ProcessOutput += err;
-        this->ProcessOutput += "\n";
-        }
-      if(gotStdOut)
-        {
-        cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
-                   this->GetIndex() << ": " << out << std::endl);
-        this->ProcessOutput += out;
-        this->ProcessOutput += "\n";
-        }
-      if(pipe == cmsysProcess_Pipe_None)
-        {
-        break;
-        }
+      // Process has terminated and all output read.
+      return false;
+      }
+    else if(p == cmsysProcess_Pipe_STDOUT ||
+            p == cmsysProcess_Pipe_STDERR)
+      {
+      // Store this line of output.
+      cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                 this->GetIndex() << ": " << line << std::endl);
+      this->ProcessOutput += line;
+      this->ProcessOutput += "\n";
       }
-    gotStdOut = false;
-    gotStdErr = false;
-    //timeout while reading process output (could denote infinite output)
-    if(cmSystemTools::GetTime() - clock_start > .1)
+    else // if(p == cmsysProcess_Pipe_Timeout)
       {
       break;
       }
     }
+  return true;
 }
 
 //---------------------------------------------------------

Source/CTest/cmCTestRunTest.h

@@ -47,8 +47,9 @@ public:
   cmCTestTestHandler::cmCTestTestResult GetTestResults()
   { return this->TestResult; }
 
-  bool IsRunning();
-  void CheckOutput();
+  // Read and store output.  Returns true if it must be called again.
+  bool CheckOutput();
+
   //launch the test process, return whether it started correctly
   bool StartTest();
   //capture and report the test results

Source/CTest/cmProcess.cxx

@@ -23,7 +23,6 @@ cmProcess::cmProcess()
   this->Process = 0;
   this->Timeout = 0;
   this->TotalTime = 0;
-  this->LastOutputPipe = cmsysProcess_Pipe_None;
   this->ExitValue = 0;
   this->Id = 0;
   this->StartTime = 0;
@@ -74,153 +73,112 @@ bool cmProcess::StartProcess()
           == cmsysProcess_State_Executing);
 }
 
-int cmProcess::GetNextOutputLine(std::string& stdOutLine,
-                            std::string& stdErrLine,
-                            bool& gotStdOut,
-                            bool& gotStdErr,
-                            bool running)
+//----------------------------------------------------------------------------
+bool cmProcess::Buffer::GetLine(std::string& line)
 {
-  if(this->StdErrorBuffer.empty() && this->StdOutBuffer.empty())
+  // Scan for the next newline.
+  for(size_type sz = this->size(); this->Last != sz; ++this->Last)
     {
-    return cmsysProcess_Pipe_Timeout;
-    }
-  stdOutLine = "";
-  stdErrLine = "";
+    if((*this)[this->Last] == '\n' || (*this)[this->Last] == '\0')
+      {
+      // Extract the range first..last as a line.
+      const char* data = &*this->begin() + this->First;
+      size_type length = this->Last - this->First;
+      length -= (length && data[length-1] == '\r')? 1:0;
+      line.assign(data, length);
 
-  this->LastOutputPipe = cmsysProcess_Pipe_Timeout;
-  std::vector<char>::iterator outiter = 
-    this->StdOutBuffer.begin();
-  std::vector<char>::iterator erriter = 
-    this->StdErrorBuffer.begin();
+      // Start a new range for the next line.
+      ++this->Last;
+      this->First = Last;
 
-  // Check for a newline in stdout.
-  for(;outiter != this->StdOutBuffer.end(); ++outiter)
-    {
-    if((*outiter == '\r') && ((outiter+1) == this->StdOutBuffer.end()))
-      {
-      break;
-      }
-    else if(*outiter == '\n' || *outiter == '\0')
-      {
-      int length = outiter-this->StdOutBuffer.begin();
-      if(length > 1 && *(outiter-1) == '\r')
-        {
-        --length;
-        }
-      if(length > 0)
-        {
-        stdOutLine.append(&this->StdOutBuffer[0], length);
-        }
-      this->StdOutBuffer.erase(this->StdOutBuffer.begin(), outiter+1);
-      this->LastOutputPipe = cmsysProcess_Pipe_STDOUT;
-      gotStdOut = true;
-      break;
+      // Return the line extracted.
+      return true;
       }
     }
 
-  // Check for a newline in stderr.
-  for(;erriter != this->StdErrorBuffer.end(); ++erriter)
+  // Available data have been exhausted without a newline.
+  if(this->First != 0)
     {
-    if((*erriter == '\r') && ((erriter+1) == this->StdErrorBuffer.end()))
-      {
-      break;
-      }
-    else if(*erriter == '\n' || *erriter == '\0')
-      {
-      int length = erriter-this->StdErrorBuffer.begin();
-      if(length > 1 && *(erriter-1) == '\r')
-        {
-        --length;
-        }
-      if(length > 0)
-        {
-        stdErrLine.append(&this->StdErrorBuffer[0], length);
-        }
-      this->StdErrorBuffer.erase(this->StdErrorBuffer.begin(), erriter+1);
-      this->LastOutputPipe = cmsysProcess_Pipe_STDERR;
-      gotStdErr = true;
-      break;
-      }
+    // Move the partial line to the beginning of the buffer.
+    this->erase(this->begin(), this->begin() + this->First);
+    this->First = 0;
+    this->Last = this->size();
     }
+  return false;
+}
 
-  if(!running && !gotStdErr && !gotStdOut)
-  {
-    //If process terminated with no newline, flush the buffer
-    if(!running)
-      {
-      if(!this->StdErrorBuffer.empty())
-        {
-        gotStdErr = true;
-        stdErrLine.append(&this->StdErrorBuffer[0],
-                          this->StdErrorBuffer.size());
-        this->StdErrorBuffer.erase(this->StdErrorBuffer.begin(), 
-          this->StdErrorBuffer.end());
-        }
-      if(!this->StdOutBuffer.empty())
-        {
-        gotStdOut = true;
-        stdOutLine.append(&this->StdOutBuffer[0],
-                          this->StdOutBuffer.size());
-        this->StdOutBuffer.erase(this->StdOutBuffer.begin(),
-          this->StdOutBuffer.end());
-        }
-      return cmsysProcess_Pipe_None;
-      }
+//----------------------------------------------------------------------------
+bool cmProcess::Buffer::GetLast(std::string& line)
+{
+  // Return the partial last line, if any.
+  if(!this->empty())
+    {
+    line.assign(&*this->begin(), this->size());
+    this->clear();
+    return true;
     }
-  //If we get here, we have stuff waiting in the buffers, but no newline
-  return this->LastOutputPipe;
+  return false;
 }
-// return true if there is a new line of data
-// return false if there is no new data
-bool cmProcess::CheckOutput(double timeout)
-{
-  // Wait for data from the process.
-  int length;
-  char* data;
 
-  while(1)
+//----------------------------------------------------------------------------
+int cmProcess::GetNextOutputLine(std::string& line, double timeout)
+{
+  for(;;)
     {
-    int pipe = cmsysProcess_WaitForData(this->Process, &data, 
-                                        &length, &timeout);
-    if(pipe == cmsysProcess_Pipe_Timeout)
+    // Look for lines already buffered.
+    if(this->StdOut.GetLine(line))
       {
-      // Timeout has been exceeded.
-      this->LastOutputPipe = pipe;
-      return true;
+      return cmsysProcess_Pipe_STDOUT;
       }
-    else if(pipe == cmsysProcess_Pipe_STDOUT)
+    else if(this->StdErr.GetLine(line))
       {
-      // Append to the stdout buffer.
-      this->StdOutBuffer.insert(this->StdOutBuffer.end(), data, data+length);
-      this->LastOutputPipe = pipe;
+      return cmsysProcess_Pipe_STDERR;
       }
-    else if(pipe == cmsysProcess_Pipe_STDERR)
+
+    // Check for more data from the process.
+    char* data;
+    int length;
+    int p = cmsysProcess_WaitForData(this->Process, &data, &length, &timeout);
+    if(p == cmsysProcess_Pipe_Timeout)
       {
-      // Append to the stderr buffer.
-      this->StdErrorBuffer.insert(this->StdErrorBuffer.end(),
-                                  data, data+length);
-      this->LastOutputPipe = pipe;
+      return cmsysProcess_Pipe_Timeout;
       }
-    else if(pipe == cmsysProcess_Pipe_None)
+    else if(p == cmsysProcess_Pipe_STDOUT)
       {
-      // Both stdout and stderr pipes have broken.  Return leftover data.
-      if(!this->StdOutBuffer.empty())
-        {
-        this->LastOutputPipe = cmsysProcess_Pipe_STDOUT;
-        return false;
-        }
-      else if(!this->StdErrorBuffer.empty())
-        {
-        this->LastOutputPipe = cmsysProcess_Pipe_STDERR;
-        return false;
-        }
-      else
-        {
-        this->LastOutputPipe = cmsysProcess_Pipe_None;
-        return false;
-        }
+      this->StdOut.insert(this->StdOut.end(), data, data+length);
+      }
+    else if(p == cmsysProcess_Pipe_STDERR)
+      {
+      this->StdErr.insert(this->StdErr.end(), data, data+length);
+      }
+    else // p == cmsysProcess_Pipe_None
+      {
+      // The process will provide no more data.
+      break;
       }
     }
+
+  // Look for partial last lines.
+  if(this->StdOut.GetLast(line))
+    {
+    return cmsysProcess_Pipe_STDOUT;
+    }
+  else if(this->StdErr.GetLast(line))
+    {
+    return cmsysProcess_Pipe_STDERR;
+    }
+
+  // No more data.  Wait for process exit.
+  if(!cmsysProcess_WaitForExit(this->Process, &timeout))
+    {
+    return cmsysProcess_Pipe_Timeout;
+    }
+
+  // Record exit information.
+  this->ExitValue = cmsysProcess_GetExitValue(this->Process);
+  this->TotalTime = cmSystemTools::GetTime() - this->StartTime;
+  //  std::cerr << "Time to run: " << this->TotalTime << "\n";
+  return cmsysProcess_Pipe_None;
 }
 
 // return the process status
@@ -233,26 +191,6 @@ int cmProcess::GetProcessStatus()
   return cmsysProcess_GetState(this->Process);
 }
 
-// return true if the process is running
-bool cmProcess::IsRunning()
-{
-  int status = this->GetProcessStatus();
-  if(status == cmsysProcess_State_Executing )
-    {
-    if(this->LastOutputPipe != 0)
-      {
-      return true;
-      }
-    }
-  // if the process is done, then wait for it to exit
-  cmsysProcess_WaitForExit(this->Process, 0);
-  this->ExitValue = cmsysProcess_GetExitValue(this->Process);
-  this->TotalTime = cmSystemTools::GetTime() - this->StartTime;
-//  std::cerr << "Time to run: " << this->TotalTime << "\n";
-  return false;
-}
-
-
 int cmProcess::ReportStatus()
 {
   int result = 1;

Source/CTest/cmProcess.h

@@ -39,29 +39,42 @@ public:
   void SetTimeout(double t) { this->Timeout = t;}
   // Return true if the process starts
   bool StartProcess();
-  
-  // return false if process has exited, true otherwise
-  bool CheckOutput(double timeout);
+
   // return the process status
   int GetProcessStatus();
-  // return true if the process is running
-  bool IsRunning();
   // Report the status of the program 
   int ReportStatus();
   int GetId() { return this->Id; }
   void SetId(int id) { this->Id = id;}
   int GetExitValue() { return this->ExitValue;}
   double GetTotalTime() { return this->TotalTime;}
-  int GetNextOutputLine(std::string& stdOutLine, std::string& stdErrLine,
-                        bool& gotStdOut, bool& gotStdErr, bool running);
+
+  /**
+   * Read one line of output but block for no more than timeout.
+   * Returns:
+   *   cmsysProcess_Pipe_None    = Process terminated and all output read
+   *   cmsysProcess_Pipe_STDOUT  = Line came from stdout
+   *   cmsysProcess_Pipe_STDOUT  = Line came from stderr
+   *   cmsysProcess_Pipe_Timeout = Timeout expired while waiting
+   */
+  int GetNextOutputLine(std::string& line, double timeout);
 private:
-  int LastOutputPipe;
   double Timeout;
   double StartTime;
   double TotalTime;
   cmsysProcess* Process;
-  std::vector<char> StdErrorBuffer;
-  std::vector<char> StdOutBuffer;
+  class Buffer: public std::vector<char>
+  {
+    // Half-open index range of partial line already scanned.
+    size_type First;
+    size_type Last;
+  public:
+    Buffer(): First(0), Last(0) {}
+    bool GetLine(std::string& line);
+    bool GetLast(std::string& line);
+  };
+  Buffer StdErr;
+  Buffer StdOut;
   std::string Command;
   std::string WorkingDirectory;
   std::vector<std::string> Arguments;