cmCTestMultiProcessHandler.cxx 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829
  1. /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
  2. file Copyright.txt or https://cmake.org/licensing for details. */
  3. #include "cmCTestMultiProcessHandler.h"
  4. #include "cmCTest.h"
  5. #include "cmCTestRunTest.h"
  6. #include "cmCTestScriptHandler.h"
  7. #include "cmCTestTestHandler.h"
  8. #include "cmSystemTools.h"
  9. #include "cmWorkingDirectory.h"
  10. #include "cmsys/FStream.hxx"
  11. #include "cmsys/String.hxx"
  12. #include "cmsys/SystemInformation.hxx"
  13. #include <algorithm>
  14. #include <chrono>
  15. #include <iomanip>
  16. #include <list>
  17. #include <math.h>
  18. #include <sstream>
  19. #include <stack>
  20. #include <stdlib.h>
  21. #include <utility>
  22. class TestComparator
  23. {
  24. public:
  25. TestComparator(cmCTestMultiProcessHandler* handler)
  26. : Handler(handler)
  27. {
  28. }
  29. ~TestComparator() {}
  30. // Sorts tests in descending order of cost
  31. bool operator()(int index1, int index2) const
  32. {
  33. return Handler->Properties[index1]->Cost >
  34. Handler->Properties[index2]->Cost;
  35. }
  36. private:
  37. cmCTestMultiProcessHandler* Handler;
  38. };
  39. cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
  40. {
  41. this->ParallelLevel = 1;
  42. this->TestLoad = 0;
  43. this->Completed = 0;
  44. this->RunningCount = 0;
  45. this->StopTimePassed = false;
  46. this->HasCycles = false;
  47. this->SerialTestRunning = false;
  48. }
  49. cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler()
  50. {
  51. }
  52. // Set the tests
  53. void cmCTestMultiProcessHandler::SetTests(TestMap& tests,
  54. PropertiesMap& properties)
  55. {
  56. this->Tests = tests;
  57. this->Properties = properties;
  58. this->Total = this->Tests.size();
  59. // set test run map to false for all
  60. for (auto const& t : this->Tests) {
  61. this->TestRunningMap[t.first] = false;
  62. this->TestFinishMap[t.first] = false;
  63. }
  64. if (!this->CTest->GetShowOnly()) {
  65. this->ReadCostData();
  66. this->HasCycles = !this->CheckCycles();
  67. if (this->HasCycles) {
  68. return;
  69. }
  70. this->CreateTestCostList();
  71. }
  72. }
  73. // Set the max number of tests that can be run at the same time.
  74. void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
  75. {
  76. this->ParallelLevel = level < 1 ? 1 : level;
  77. }
  78. void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
  79. {
  80. this->TestLoad = load;
  81. }
  82. void cmCTestMultiProcessHandler::RunTests()
  83. {
  84. this->CheckResume();
  85. if (this->HasCycles) {
  86. return;
  87. }
  88. this->TestHandler->SetMaxIndex(this->FindMaxIndex());
  89. this->StartNextTests();
  90. while (!this->Tests.empty()) {
  91. if (this->StopTimePassed) {
  92. return;
  93. }
  94. this->CheckOutput();
  95. this->StartNextTests();
  96. }
  97. // let all running tests finish
  98. while (this->CheckOutput()) {
  99. }
  100. this->MarkFinished();
  101. this->UpdateCostData();
  102. }
  103. void cmCTestMultiProcessHandler::StartTestProcess(int test)
  104. {
  105. std::chrono::system_clock::time_point stop_time = this->CTest->GetStopTime();
  106. if (stop_time != std::chrono::system_clock::time_point() &&
  107. stop_time <= std::chrono::system_clock::now()) {
  108. cmCTestLog(this->CTest, ERROR_MESSAGE, "The stop time has been passed. "
  109. "Stopping all tests."
  110. << std::endl);
  111. this->StopTimePassed = true;
  112. return;
  113. }
  114. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  115. "test " << test << "\n", this->Quiet);
  116. this->TestRunningMap[test] = true; // mark the test as running
  117. // now remove the test itself
  118. this->EraseTest(test);
  119. this->RunningCount += GetProcessorsUsed(test);
  120. cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
  121. if (this->CTest->GetRepeatUntilFail()) {
  122. testRun->SetRunUntilFailOn();
  123. testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
  124. }
  125. testRun->SetIndex(test);
  126. testRun->SetTestProperties(this->Properties[test]);
  127. // Find any failed dependencies for this test. We assume the more common
  128. // scenario has no failed tests, so make it the outer loop.
  129. for (std::string const& f : *this->Failed) {
  130. if (this->Properties[test]->RequireSuccessDepends.find(f) !=
  131. this->Properties[test]->RequireSuccessDepends.end()) {
  132. testRun->AddFailedDependency(f);
  133. }
  134. }
  135. cmWorkingDirectory workdir(this->Properties[test]->Directory);
  136. // Lock the resources we'll be using
  137. this->LockResources(test);
  138. if (testRun->StartTest(this->Total)) {
  139. this->RunningTests.insert(testRun);
  140. } else {
  141. for (auto& j : this->Tests) {
  142. j.second.erase(test);
  143. }
  144. this->UnlockResources(test);
  145. this->Completed++;
  146. this->TestFinishMap[test] = true;
  147. this->TestRunningMap[test] = false;
  148. this->RunningCount -= GetProcessorsUsed(test);
  149. testRun->EndTest(this->Completed, this->Total, false);
  150. if (!this->Properties[test]->Disabled) {
  151. this->Failed->push_back(this->Properties[test]->Name);
  152. }
  153. delete testRun;
  154. }
  155. }
  156. void cmCTestMultiProcessHandler::LockResources(int index)
  157. {
  158. this->LockedResources.insert(
  159. this->Properties[index]->LockedResources.begin(),
  160. this->Properties[index]->LockedResources.end());
  161. if (this->Properties[index]->RunSerial) {
  162. this->SerialTestRunning = true;
  163. }
  164. }
  165. void cmCTestMultiProcessHandler::UnlockResources(int index)
  166. {
  167. for (std::string const& i : this->Properties[index]->LockedResources) {
  168. this->LockedResources.erase(i);
  169. }
  170. if (this->Properties[index]->RunSerial) {
  171. this->SerialTestRunning = false;
  172. }
  173. }
  174. void cmCTestMultiProcessHandler::EraseTest(int test)
  175. {
  176. this->Tests.erase(test);
  177. this->SortedTests.erase(
  178. std::find(this->SortedTests.begin(), this->SortedTests.end(), test));
  179. }
  180. inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
  181. {
  182. size_t processors = static_cast<int>(this->Properties[test]->Processors);
  183. // If processors setting is set higher than the -j
  184. // setting, we default to using all of the process slots.
  185. if (processors > this->ParallelLevel) {
  186. processors = this->ParallelLevel;
  187. }
  188. return processors;
  189. }
  190. std::string cmCTestMultiProcessHandler::GetName(int test)
  191. {
  192. return this->Properties[test]->Name;
  193. }
  194. bool cmCTestMultiProcessHandler::StartTest(int test)
  195. {
  196. // Check for locked resources
  197. for (std::string const& i : this->Properties[test]->LockedResources) {
  198. if (this->LockedResources.find(i) != this->LockedResources.end()) {
  199. return false;
  200. }
  201. }
  202. // if there are no depends left then run this test
  203. if (this->Tests[test].empty()) {
  204. this->StartTestProcess(test);
  205. return true;
  206. }
  207. // This test was not able to start because it is waiting
  208. // on depends to run
  209. return false;
  210. }
  211. void cmCTestMultiProcessHandler::StartNextTests()
  212. {
  213. size_t numToStart = 0;
  214. if (this->RunningCount < this->ParallelLevel) {
  215. numToStart = this->ParallelLevel - this->RunningCount;
  216. }
  217. if (numToStart == 0) {
  218. return;
  219. }
  220. // Don't start any new tests if one with the RUN_SERIAL property
  221. // is already running.
  222. if (this->SerialTestRunning) {
  223. return;
  224. }
  225. bool allTestsFailedTestLoadCheck = false;
  226. bool usedFakeLoadForTesting = false;
  227. size_t minProcessorsRequired = this->ParallelLevel;
  228. std::string testWithMinProcessors;
  229. cmsys::SystemInformation info;
  230. unsigned long systemLoad = 0;
  231. size_t spareLoad = 0;
  232. if (this->TestLoad > 0) {
  233. // Activate possible wait.
  234. allTestsFailedTestLoadCheck = true;
  235. // Check for a fake load average value used in testing.
  236. std::string fake_load_value;
  237. if (cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING",
  238. fake_load_value)) {
  239. usedFakeLoadForTesting = true;
  240. if (!cmSystemTools::StringToULong(fake_load_value.c_str(),
  241. &systemLoad)) {
  242. cmSystemTools::Error("Failed to parse fake load value: ",
  243. fake_load_value.c_str());
  244. }
  245. }
  246. // If it's not set, look up the true load average.
  247. else {
  248. systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
  249. }
  250. spareLoad =
  251. (this->TestLoad > systemLoad ? this->TestLoad - systemLoad : 0);
  252. // Don't start more tests than the spare load can support.
  253. if (numToStart > spareLoad) {
  254. numToStart = spareLoad;
  255. }
  256. }
  257. TestList copy = this->SortedTests;
  258. for (auto const& test : copy) {
  259. // Take a nap if we're currently performing a RUN_SERIAL test.
  260. if (this->SerialTestRunning) {
  261. break;
  262. }
  263. // We can only start a RUN_SERIAL test if no other tests are also running.
  264. if (this->Properties[test]->RunSerial && this->RunningCount > 0) {
  265. continue;
  266. }
  267. size_t processors = GetProcessorsUsed(test);
  268. bool testLoadOk = true;
  269. if (this->TestLoad > 0) {
  270. if (processors <= spareLoad) {
  271. cmCTestLog(this->CTest, DEBUG, "OK to run "
  272. << GetName(test) << ", it requires " << processors
  273. << " procs & system load is: " << systemLoad
  274. << std::endl);
  275. allTestsFailedTestLoadCheck = false;
  276. } else {
  277. testLoadOk = false;
  278. }
  279. }
  280. if (processors <= minProcessorsRequired) {
  281. minProcessorsRequired = processors;
  282. testWithMinProcessors = GetName(test);
  283. }
  284. if (testLoadOk && processors <= numToStart && this->StartTest(test)) {
  285. if (this->StopTimePassed) {
  286. return;
  287. }
  288. numToStart -= processors;
  289. } else if (numToStart == 0) {
  290. break;
  291. }
  292. }
  293. if (allTestsFailedTestLoadCheck) {
  294. // Find out whether there are any non RUN_SERIAL tests left, so that the
  295. // correct warning may be displayed.
  296. bool onlyRunSerialTestsLeft = true;
  297. for (auto const& test : copy) {
  298. if (!this->Properties[test]->RunSerial) {
  299. onlyRunSerialTestsLeft = false;
  300. }
  301. }
  302. cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, ");
  303. if (this->SerialTestRunning) {
  304. cmCTestLog(this->CTest, HANDLER_OUTPUT,
  305. "Waiting for RUN_SERIAL test to finish.");
  306. } else if (onlyRunSerialTestsLeft) {
  307. cmCTestLog(this->CTest, HANDLER_OUTPUT,
  308. "Only RUN_SERIAL tests remain, awaiting available slot.");
  309. } else {
  310. /* clang-format off */
  311. cmCTestLog(this->CTest, HANDLER_OUTPUT,
  312. "System Load: " << systemLoad << ", "
  313. "Max Allowed Load: " << this->TestLoad << ", "
  314. "Smallest test " << testWithMinProcessors <<
  315. " requires " << minProcessorsRequired);
  316. /* clang-format on */
  317. }
  318. cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl);
  319. if (usedFakeLoadForTesting) {
  320. // Break out of the infinite loop of waiting for our fake load
  321. // to come down.
  322. this->StopTimePassed = true;
  323. } else {
  324. // Wait between 1 and 5 seconds before trying again.
  325. cmCTestScriptHandler::SleepInSeconds(cmSystemTools::RandomSeed() % 5 +
  326. 1);
  327. }
  328. }
  329. }
  330. bool cmCTestMultiProcessHandler::CheckOutput()
  331. {
  332. // no more output we are done
  333. if (this->RunningTests.empty()) {
  334. return false;
  335. }
  336. std::vector<cmCTestRunTest*> finished;
  337. std::string out, err;
  338. for (cmCTestRunTest* p : this->RunningTests) {
  339. if (!p->CheckOutput()) {
  340. finished.push_back(p);
  341. }
  342. }
  343. for (cmCTestRunTest* p : finished) {
  344. this->Completed++;
  345. int test = p->GetIndex();
  346. bool testResult = p->EndTest(this->Completed, this->Total, true);
  347. if (p->StartAgain()) {
  348. this->Completed--; // remove the completed test because run again
  349. continue;
  350. }
  351. if (testResult) {
  352. this->Passed->push_back(p->GetTestProperties()->Name);
  353. } else {
  354. this->Failed->push_back(p->GetTestProperties()->Name);
  355. }
  356. for (auto& t : this->Tests) {
  357. t.second.erase(test);
  358. }
  359. this->TestFinishMap[test] = true;
  360. this->TestRunningMap[test] = false;
  361. this->RunningTests.erase(p);
  362. this->WriteCheckpoint(test);
  363. this->UnlockResources(test);
  364. this->RunningCount -= GetProcessorsUsed(test);
  365. delete p;
  366. }
  367. return true;
  368. }
  369. void cmCTestMultiProcessHandler::UpdateCostData()
  370. {
  371. std::string fname = this->CTest->GetCostDataFile();
  372. std::string tmpout = fname + ".tmp";
  373. cmsys::ofstream fout;
  374. fout.open(tmpout.c_str());
  375. PropertiesMap temp = this->Properties;
  376. if (cmSystemTools::FileExists(fname.c_str())) {
  377. cmsys::ifstream fin;
  378. fin.open(fname.c_str());
  379. std::string line;
  380. while (std::getline(fin, line)) {
  381. if (line == "---") {
  382. break;
  383. }
  384. std::vector<cmsys::String> parts = cmSystemTools::SplitString(line, ' ');
  385. // Format: <name> <previous_runs> <avg_cost>
  386. if (parts.size() < 3) {
  387. break;
  388. }
  389. std::string name = parts[0];
  390. int prev = atoi(parts[1].c_str());
  391. float cost = static_cast<float>(atof(parts[2].c_str()));
  392. int index = this->SearchByName(name);
  393. if (index == -1) {
  394. // This test is not in memory. We just rewrite the entry
  395. fout << name << " " << prev << " " << cost << "\n";
  396. } else {
  397. // Update with our new average cost
  398. fout << name << " " << this->Properties[index]->PreviousRuns << " "
  399. << this->Properties[index]->Cost << "\n";
  400. temp.erase(index);
  401. }
  402. }
  403. fin.close();
  404. cmSystemTools::RemoveFile(fname);
  405. }
  406. // Add all tests not previously listed in the file
  407. for (auto const& i : temp) {
  408. fout << i.second->Name << " " << i.second->PreviousRuns << " "
  409. << i.second->Cost << "\n";
  410. }
  411. // Write list of failed tests
  412. fout << "---\n";
  413. for (std::string const& f : *this->Failed) {
  414. fout << f << "\n";
  415. }
  416. fout.close();
  417. cmSystemTools::RenameFile(tmpout.c_str(), fname.c_str());
  418. }
  419. void cmCTestMultiProcessHandler::ReadCostData()
  420. {
  421. std::string fname = this->CTest->GetCostDataFile();
  422. if (cmSystemTools::FileExists(fname.c_str(), true)) {
  423. cmsys::ifstream fin;
  424. fin.open(fname.c_str());
  425. std::string line;
  426. while (std::getline(fin, line)) {
  427. if (line == "---") {
  428. break;
  429. }
  430. std::vector<cmsys::String> parts = cmSystemTools::SplitString(line, ' ');
  431. // Probably an older version of the file, will be fixed next run
  432. if (parts.size() < 3) {
  433. fin.close();
  434. return;
  435. }
  436. std::string name = parts[0];
  437. int prev = atoi(parts[1].c_str());
  438. float cost = static_cast<float>(atof(parts[2].c_str()));
  439. int index = this->SearchByName(name);
  440. if (index == -1) {
  441. continue;
  442. }
  443. this->Properties[index]->PreviousRuns = prev;
  444. // When not running in parallel mode, don't use cost data
  445. if (this->ParallelLevel > 1 && this->Properties[index] &&
  446. this->Properties[index]->Cost == 0) {
  447. this->Properties[index]->Cost = cost;
  448. }
  449. }
  450. // Next part of the file is the failed tests
  451. while (std::getline(fin, line)) {
  452. if (!line.empty()) {
  453. this->LastTestsFailed.push_back(line);
  454. }
  455. }
  456. fin.close();
  457. }
  458. }
  459. int cmCTestMultiProcessHandler::SearchByName(std::string const& name)
  460. {
  461. int index = -1;
  462. for (auto const& p : this->Properties) {
  463. if (p.second->Name == name) {
  464. index = p.first;
  465. }
  466. }
  467. return index;
  468. }
  469. void cmCTestMultiProcessHandler::CreateTestCostList()
  470. {
  471. if (this->ParallelLevel > 1) {
  472. CreateParallelTestCostList();
  473. } else {
  474. CreateSerialTestCostList();
  475. }
  476. }
  477. void cmCTestMultiProcessHandler::CreateParallelTestCostList()
  478. {
  479. TestSet alreadySortedTests;
  480. std::list<TestSet> priorityStack;
  481. priorityStack.push_back(TestSet());
  482. TestSet& topLevel = priorityStack.back();
  483. // In parallel test runs add previously failed tests to the front
  484. // of the cost list and queue other tests for further sorting
  485. for (auto const& t : this->Tests) {
  486. if (std::find(this->LastTestsFailed.begin(), this->LastTestsFailed.end(),
  487. this->Properties[t.first]->Name) !=
  488. this->LastTestsFailed.end()) {
  489. // If the test failed last time, it should be run first.
  490. this->SortedTests.push_back(t.first);
  491. alreadySortedTests.insert(t.first);
  492. } else {
  493. topLevel.insert(t.first);
  494. }
  495. }
  496. // In parallel test runs repeatedly move dependencies of the tests on
  497. // the current dependency level to the next level until no
  498. // further dependencies exist.
  499. while (!priorityStack.back().empty()) {
  500. TestSet& previousSet = priorityStack.back();
  501. priorityStack.push_back(TestSet());
  502. TestSet& currentSet = priorityStack.back();
  503. for (auto const& i : previousSet) {
  504. TestSet const& dependencies = this->Tests[i];
  505. currentSet.insert(dependencies.begin(), dependencies.end());
  506. }
  507. for (auto const& i : currentSet) {
  508. previousSet.erase(i);
  509. }
  510. }
  511. // Remove the empty dependency level
  512. priorityStack.pop_back();
  513. // Reverse iterate over the different dependency levels (deepest first).
  514. // Sort tests within each level by COST and append them to the cost list.
  515. for (std::list<TestSet>::reverse_iterator i = priorityStack.rbegin();
  516. i != priorityStack.rend(); ++i) {
  517. TestSet const& currentSet = *i;
  518. TestComparator comp(this);
  519. TestList sortedCopy;
  520. sortedCopy.insert(sortedCopy.end(), currentSet.begin(), currentSet.end());
  521. std::stable_sort(sortedCopy.begin(), sortedCopy.end(), comp);
  522. for (auto const& j : sortedCopy) {
  523. if (alreadySortedTests.find(j) == alreadySortedTests.end()) {
  524. this->SortedTests.push_back(j);
  525. alreadySortedTests.insert(j);
  526. }
  527. }
  528. }
  529. }
  530. void cmCTestMultiProcessHandler::GetAllTestDependencies(int test,
  531. TestList& dependencies)
  532. {
  533. TestSet const& dependencySet = this->Tests[test];
  534. for (int i : dependencySet) {
  535. GetAllTestDependencies(i, dependencies);
  536. dependencies.push_back(i);
  537. }
  538. }
  539. void cmCTestMultiProcessHandler::CreateSerialTestCostList()
  540. {
  541. TestList presortedList;
  542. for (auto const& i : this->Tests) {
  543. presortedList.push_back(i.first);
  544. }
  545. TestComparator comp(this);
  546. std::stable_sort(presortedList.begin(), presortedList.end(), comp);
  547. TestSet alreadySortedTests;
  548. for (int test : presortedList) {
  549. if (alreadySortedTests.find(test) != alreadySortedTests.end()) {
  550. continue;
  551. }
  552. TestList dependencies;
  553. GetAllTestDependencies(test, dependencies);
  554. for (int testDependency : dependencies) {
  555. if (alreadySortedTests.find(testDependency) ==
  556. alreadySortedTests.end()) {
  557. alreadySortedTests.insert(testDependency);
  558. this->SortedTests.push_back(testDependency);
  559. }
  560. }
  561. alreadySortedTests.insert(test);
  562. this->SortedTests.push_back(test);
  563. }
  564. }
  565. void cmCTestMultiProcessHandler::WriteCheckpoint(int index)
  566. {
  567. std::string fname =
  568. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  569. cmsys::ofstream fout;
  570. fout.open(fname.c_str(), std::ios::app);
  571. fout << index << "\n";
  572. fout.close();
  573. }
  574. void cmCTestMultiProcessHandler::MarkFinished()
  575. {
  576. std::string fname =
  577. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  578. cmSystemTools::RemoveFile(fname);
  579. }
  580. // For ShowOnly mode
  581. void cmCTestMultiProcessHandler::PrintTestList()
  582. {
  583. this->TestHandler->SetMaxIndex(this->FindMaxIndex());
  584. int count = 0;
  585. for (auto& it : this->Properties) {
  586. count++;
  587. cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
  588. cmWorkingDirectory workdir(p.Directory);
  589. cmCTestRunTest testRun(this->TestHandler);
  590. testRun.SetIndex(p.Index);
  591. testRun.SetTestProperties(&p);
  592. testRun.ComputeArguments(); // logs the command in verbose mode
  593. if (!p.Labels.empty()) // print the labels
  594. {
  595. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Labels:",
  596. this->Quiet);
  597. }
  598. for (std::string const& label : p.Labels) {
  599. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, " " << label,
  600. this->Quiet);
  601. }
  602. if (!p.Labels.empty()) // print the labels
  603. {
  604. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl,
  605. this->Quiet);
  606. }
  607. if (this->TestHandler->MemCheck) {
  608. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Memory Check",
  609. this->Quiet);
  610. } else {
  611. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Test", this->Quiet);
  612. }
  613. std::ostringstream indexStr;
  614. indexStr << " #" << p.Index << ":";
  615. cmCTestOptionalLog(
  616. this->CTest, HANDLER_OUTPUT,
  617. std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex()))
  618. << indexStr.str(),
  619. this->Quiet);
  620. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << p.Name,
  621. this->Quiet);
  622. if (p.Disabled) {
  623. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " (Disabled)",
  624. this->Quiet);
  625. }
  626. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, std::endl, this->Quiet);
  627. }
  628. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, std::endl
  629. << "Total Tests: " << this->Total << std::endl,
  630. this->Quiet);
  631. }
  632. void cmCTestMultiProcessHandler::PrintLabels()
  633. {
  634. std::set<std::string> allLabels;
  635. for (auto& it : this->Properties) {
  636. cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
  637. allLabels.insert(p.Labels.begin(), p.Labels.end());
  638. }
  639. if (!allLabels.empty()) {
  640. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "All Labels:" << std::endl,
  641. this->Quiet);
  642. } else {
  643. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
  644. "No Labels Exist" << std::endl, this->Quiet);
  645. }
  646. for (std::string const& label : allLabels) {
  647. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << label << std::endl,
  648. this->Quiet);
  649. }
  650. }
  651. void cmCTestMultiProcessHandler::CheckResume()
  652. {
  653. std::string fname =
  654. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  655. if (this->CTest->GetFailover()) {
  656. if (cmSystemTools::FileExists(fname.c_str(), true)) {
  657. *this->TestHandler->LogFile
  658. << "Resuming previously interrupted test set" << std::endl
  659. << "----------------------------------------------------------"
  660. << std::endl;
  661. cmsys::ifstream fin;
  662. fin.open(fname.c_str());
  663. std::string line;
  664. while (std::getline(fin, line)) {
  665. int index = atoi(line.c_str());
  666. this->RemoveTest(index);
  667. }
  668. fin.close();
  669. }
  670. } else if (cmSystemTools::FileExists(fname.c_str(), true)) {
  671. cmSystemTools::RemoveFile(fname);
  672. }
  673. }
  674. void cmCTestMultiProcessHandler::RemoveTest(int index)
  675. {
  676. this->EraseTest(index);
  677. this->Properties.erase(index);
  678. this->TestRunningMap[index] = false;
  679. this->TestFinishMap[index] = true;
  680. this->Completed++;
  681. }
  682. int cmCTestMultiProcessHandler::FindMaxIndex()
  683. {
  684. int max = 0;
  685. for (auto const& i : this->Tests) {
  686. if (i.first > max) {
  687. max = i.first;
  688. }
  689. }
  690. return max;
  691. }
  692. // Returns true if no cycles exist in the dependency graph
  693. bool cmCTestMultiProcessHandler::CheckCycles()
  694. {
  695. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  696. "Checking test dependency graph..." << std::endl,
  697. this->Quiet);
  698. for (auto const& it : this->Tests) {
  699. // DFS from each element to itself
  700. int root = it.first;
  701. std::set<int> visited;
  702. std::stack<int> s;
  703. s.push(root);
  704. while (!s.empty()) {
  705. int test = s.top();
  706. s.pop();
  707. if (visited.insert(test).second) {
  708. for (auto const& d : this->Tests[test]) {
  709. if (d == root) {
  710. // cycle exists
  711. cmCTestLog(
  712. this->CTest, ERROR_MESSAGE,
  713. "Error: a cycle exists in the test dependency graph "
  714. "for the test \""
  715. << this->Properties[root]->Name
  716. << "\".\nPlease fix the cycle and run ctest again.\n");
  717. return false;
  718. }
  719. s.push(d);
  720. }
  721. }
  722. }
  723. }
  724. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  725. "Checking test dependency graph end" << std::endl,
  726. this->Quiet);
  727. return true;
  728. }