cmCTestMultiProcessHandler.cxx 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574
  1. /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
  2. file Copyright.txt or https://cmake.org/licensing for details. */
  3. #include "cmCTestMultiProcessHandler.h"
  4. #include <algorithm>
  5. #include <cassert>
  6. #include <chrono>
  7. #include <cmath>
  8. #include <cstddef> // IWYU pragma: keep
  9. #include <cstdlib>
  10. #include <cstring>
  11. #include <iomanip>
  12. #include <iostream>
  13. #include <list>
  14. #include <sstream>
  15. #include <stack>
  16. #include <unordered_map>
  17. #include <utility>
  18. #include <vector>
  19. #include <cm/memory>
  20. #include <cm/optional>
  21. #include <cmext/algorithm>
  22. #include <cm3p/json/value.h>
  23. #include <cm3p/json/writer.h>
  24. #include <cm3p/uv.h>
  25. #include "cmsys/FStream.hxx"
  26. #include "cmsys/SystemInformation.hxx"
  27. #include "cmAffinity.h"
  28. #include "cmCTest.h"
  29. #include "cmCTestBinPacker.h"
  30. #include "cmCTestRunTest.h"
  31. #include "cmCTestTestHandler.h"
  32. #include "cmDuration.h"
  33. #include "cmJSONState.h"
  34. #include "cmListFileCache.h"
  35. #include "cmRange.h"
  36. #include "cmStringAlgorithms.h"
  37. #include "cmSystemTools.h"
  38. #include "cmUVJobServerClient.h"
  39. #include "cmWorkingDirectory.h"
  40. namespace cmsys {
  41. class RegularExpression;
  42. }
  43. class TestComparator
  44. {
  45. public:
  46. TestComparator(cmCTestMultiProcessHandler* handler)
  47. : Handler(handler)
  48. {
  49. }
  50. // Sorts tests in descending order of cost
  51. bool operator()(int index1, int index2) const
  52. {
  53. return this->Handler->Properties[index1]->Cost >
  54. this->Handler->Properties[index2]->Cost;
  55. }
  56. private:
  57. cmCTestMultiProcessHandler* Handler;
  58. };
  59. cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
  60. {
  61. this->ParallelLevel = 1;
  62. this->TestLoad = 0;
  63. this->FakeLoadForTesting = 0;
  64. this->Completed = 0;
  65. this->RunningCount = 0;
  66. this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable();
  67. this->HaveAffinity = this->ProcessorsAvailable.size();
  68. this->HasCycles = false;
  69. this->HasInvalidGeneratedResourceSpec = false;
  70. this->SerialTestRunning = false;
  71. }
  72. cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
  73. // Set the tests
  74. void cmCTestMultiProcessHandler::SetTests(TestMap tests,
  75. PropertiesMap properties)
  76. {
  77. this->PendingTests = std::move(tests);
  78. this->Properties = std::move(properties);
  79. this->Total = this->PendingTests.size();
  80. if (!this->CTest->GetShowOnly()) {
  81. this->ReadCostData();
  82. this->HasCycles = !this->CheckCycles();
  83. this->HasInvalidGeneratedResourceSpec =
  84. !this->CheckGeneratedResourceSpec();
  85. if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
  86. return;
  87. }
  88. this->CreateTestCostList();
  89. }
  90. }
  91. // Set the max number of tests that can be run at the same time.
  92. void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
  93. {
  94. this->ParallelLevel = level < 1 ? 1 : level;
  95. }
  96. void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
  97. {
  98. this->TestLoad = load;
  99. std::string fake_load_value;
  100. if (cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING",
  101. fake_load_value)) {
  102. if (!cmStrToULong(fake_load_value, &this->FakeLoadForTesting)) {
  103. cmSystemTools::Error("Failed to parse fake load value: " +
  104. fake_load_value);
  105. }
  106. }
  107. }
  108. bool cmCTestMultiProcessHandler::Complete()
  109. {
  110. return this->Completed == this->Total;
  111. }
  112. void cmCTestMultiProcessHandler::InitializeLoop()
  113. {
  114. this->Loop.init();
  115. this->StartNextTestsOnIdle_.init(*this->Loop, this);
  116. this->StartNextTestsOnTimer_.init(*this->Loop, this);
  117. this->JobServerClient = cmUVJobServerClient::Connect(
  118. *this->Loop, /*onToken=*/[this]() { this->JobServerReceivedToken(); },
  119. /*onDisconnect=*/nullptr);
  120. if (this->JobServerClient) {
  121. cmCTestLog(this->CTest, OUTPUT,
  122. "Connected to MAKE jobserver" << std::endl);
  123. }
  124. }
  125. void cmCTestMultiProcessHandler::FinalizeLoop()
  126. {
  127. this->JobServerClient.reset();
  128. this->StartNextTestsOnTimer_.reset();
  129. this->StartNextTestsOnIdle_.reset();
  130. this->Loop.reset();
  131. }
  132. void cmCTestMultiProcessHandler::RunTests()
  133. {
  134. this->CheckResume();
  135. if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
  136. return;
  137. }
  138. this->TestHandler->SetMaxIndex(this->FindMaxIndex());
  139. this->InitializeLoop();
  140. this->StartNextTestsOnIdle();
  141. uv_run(this->Loop, UV_RUN_DEFAULT);
  142. this->FinalizeLoop();
  143. if (!this->StopTimePassed && !this->CheckStopOnFailure()) {
  144. assert(this->Complete());
  145. assert(this->PendingTests.empty());
  146. }
  147. assert(this->AllResourcesAvailable());
  148. this->MarkFinished();
  149. this->UpdateCostData();
  150. }
  151. void cmCTestMultiProcessHandler::StartTestProcess(int test)
  152. {
  153. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  154. "test " << test << "\n", this->Quiet);
  155. auto testRun = cm::make_unique<cmCTestRunTest>(*this, test);
  156. if (this->RepeatMode != cmCTest::Repeat::Never) {
  157. testRun->SetRepeatMode(this->RepeatMode);
  158. testRun->SetNumberOfRuns(this->RepeatCount);
  159. }
  160. if (this->UseResourceSpec) {
  161. testRun->SetUseAllocatedResources(true);
  162. testRun->SetAllocatedResources(this->AllocatedResources[test]);
  163. }
  164. // Find any failed dependencies for this test. We assume the more common
  165. // scenario has no failed tests, so make it the outer loop.
  166. for (std::string const& f : *this->Failed) {
  167. if (cm::contains(this->Properties[test]->RequireSuccessDepends, f)) {
  168. testRun->AddFailedDependency(f);
  169. }
  170. }
  171. if (!this->ResourceAvailabilityErrors[test].empty()) {
  172. std::ostringstream e;
  173. e << "Insufficient resources for test " << this->Properties[test]->Name
  174. << ":\n\n";
  175. for (auto const& it : this->ResourceAvailabilityErrors[test]) {
  176. switch (it.second) {
  177. case ResourceAvailabilityError::NoResourceType:
  178. e << " Test requested resources of type '" << it.first
  179. << "' which does not exist\n";
  180. break;
  181. case ResourceAvailabilityError::InsufficientResources:
  182. e << " Test requested resources of type '" << it.first
  183. << "' in the following amounts:\n";
  184. for (auto const& group : this->Properties[test]->ResourceGroups) {
  185. for (auto const& requirement : group) {
  186. if (requirement.ResourceType == it.first) {
  187. e << " " << requirement.SlotsNeeded
  188. << (requirement.SlotsNeeded == 1 ? " slot\n" : " slots\n");
  189. }
  190. }
  191. }
  192. e << " but only the following units were available:\n";
  193. for (auto const& res :
  194. this->ResourceAllocator.GetResources().at(it.first)) {
  195. e << " '" << res.first << "': " << res.second.Total
  196. << (res.second.Total == 1 ? " slot\n" : " slots\n");
  197. }
  198. break;
  199. }
  200. e << "\n";
  201. }
  202. e << "Resource spec file:\n\n " << this->ResourceSpecFile;
  203. cmCTestRunTest::StartFailure(std::move(testRun), this->Total, e.str(),
  204. "Insufficient resources");
  205. return;
  206. }
  207. cmWorkingDirectory workdir(this->Properties[test]->Directory);
  208. if (workdir.Failed()) {
  209. cmCTestRunTest::StartFailure(std::move(testRun), this->Total,
  210. "Failed to change working directory to " +
  211. this->Properties[test]->Directory + " : " +
  212. std::strerror(workdir.GetLastResult()),
  213. "Failed to change working directory");
  214. return;
  215. }
  216. // Ownership of 'testRun' has moved to another structure.
  217. // When the test finishes, FinishTestProcess will be called.
  218. cmCTestRunTest::StartTest(std::move(testRun), this->Completed, this->Total);
  219. }
  220. bool cmCTestMultiProcessHandler::AllocateResources(int index)
  221. {
  222. if (!this->UseResourceSpec) {
  223. return true;
  224. }
  225. // If the test needs unavailable resources then do not allocate anything
  226. // because it will never run. We will issue the recorded errors instead.
  227. if (!this->ResourceAvailabilityErrors[index].empty()) {
  228. return true;
  229. }
  230. std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
  231. if (!this->TryAllocateResources(index, allocations)) {
  232. return false;
  233. }
  234. auto& allocatedResources = this->AllocatedResources[index];
  235. allocatedResources.resize(this->Properties[index]->ResourceGroups.size());
  236. for (auto const& it : allocations) {
  237. for (auto const& alloc : it.second) {
  238. bool result = this->ResourceAllocator.AllocateResource(
  239. it.first, alloc.Id, alloc.SlotsNeeded);
  240. (void)result;
  241. assert(result);
  242. allocatedResources[alloc.ProcessIndex][it.first].push_back(
  243. { alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
  244. }
  245. }
  246. return true;
  247. }
  248. bool cmCTestMultiProcessHandler::TryAllocateResources(
  249. int index,
  250. std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations,
  251. std::map<std::string, ResourceAvailabilityError>* errors)
  252. {
  253. allocations.clear();
  254. std::size_t processIndex = 0;
  255. for (auto const& process : this->Properties[index]->ResourceGroups) {
  256. for (auto const& requirement : process) {
  257. for (int i = 0; i < requirement.UnitsNeeded; ++i) {
  258. allocations[requirement.ResourceType].push_back(
  259. { processIndex, requirement.SlotsNeeded, "" });
  260. }
  261. }
  262. ++processIndex;
  263. }
  264. bool result = true;
  265. auto const& availableResources = this->ResourceAllocator.GetResources();
  266. for (auto& it : allocations) {
  267. if (!availableResources.count(it.first)) {
  268. if (errors) {
  269. (*errors)[it.first] = ResourceAvailabilityError::NoResourceType;
  270. result = false;
  271. } else {
  272. return false;
  273. }
  274. } else if (!cmAllocateCTestResourcesRoundRobin(
  275. availableResources.at(it.first), it.second)) {
  276. if (errors) {
  277. (*errors)[it.first] = ResourceAvailabilityError::InsufficientResources;
  278. result = false;
  279. } else {
  280. return false;
  281. }
  282. }
  283. }
  284. return result;
  285. }
  286. void cmCTestMultiProcessHandler::DeallocateResources(int index)
  287. {
  288. if (!this->UseResourceSpec) {
  289. return;
  290. }
  291. {
  292. auto& allocatedResources = this->AllocatedResources[index];
  293. for (auto const& processAlloc : allocatedResources) {
  294. for (auto const& it : processAlloc) {
  295. auto resourceType = it.first;
  296. for (auto const& it2 : it.second) {
  297. bool success = this->ResourceAllocator.DeallocateResource(
  298. resourceType, it2.Id, it2.Slots);
  299. (void)success;
  300. assert(success);
  301. }
  302. }
  303. }
  304. }
  305. this->AllocatedResources.erase(index);
  306. }
  307. bool cmCTestMultiProcessHandler::AllResourcesAvailable()
  308. {
  309. for (auto const& it : this->ResourceAllocator.GetResources()) {
  310. for (auto const& it2 : it.second) {
  311. if (it2.second.Locked != 0) {
  312. return false;
  313. }
  314. }
  315. }
  316. return true;
  317. }
  318. void cmCTestMultiProcessHandler::CheckResourceAvailability()
  319. {
  320. if (this->UseResourceSpec) {
  321. for (auto const& t : this->PendingTests) {
  322. std::map<std::string, std::vector<cmCTestBinPackerAllocation>>
  323. allocations;
  324. this->TryAllocateResources(t.first, allocations,
  325. &this->ResourceAvailabilityErrors[t.first]);
  326. }
  327. }
  328. }
  329. bool cmCTestMultiProcessHandler::CheckStopOnFailure()
  330. {
  331. return this->CTest->GetStopOnFailure();
  332. }
  333. bool cmCTestMultiProcessHandler::CheckStopTimePassed()
  334. {
  335. if (!this->StopTimePassed) {
  336. std::chrono::system_clock::time_point stop_time =
  337. this->CTest->GetStopTime();
  338. if (stop_time != std::chrono::system_clock::time_point() &&
  339. stop_time <= std::chrono::system_clock::now()) {
  340. this->SetStopTimePassed();
  341. }
  342. }
  343. return this->StopTimePassed;
  344. }
  345. void cmCTestMultiProcessHandler::SetStopTimePassed()
  346. {
  347. if (!this->StopTimePassed) {
  348. cmCTestLog(this->CTest, ERROR_MESSAGE,
  349. "The stop time has been passed. "
  350. "Stopping all tests."
  351. << std::endl);
  352. this->StopTimePassed = true;
  353. }
  354. }
  355. void cmCTestMultiProcessHandler::LockResources(int index)
  356. {
  357. this->RunningCount += this->GetProcessorsUsed(index);
  358. auto* properties = this->Properties[index];
  359. this->ProjectResourcesLocked.insert(properties->ProjectResources.begin(),
  360. properties->ProjectResources.end());
  361. if (properties->RunSerial) {
  362. this->SerialTestRunning = true;
  363. }
  364. if (this->HaveAffinity && properties->WantAffinity) {
  365. size_t needProcessors = this->GetProcessorsUsed(index);
  366. assert(needProcessors <= this->ProcessorsAvailable.size());
  367. std::vector<size_t> affinity;
  368. affinity.reserve(needProcessors);
  369. for (size_t i = 0; i < needProcessors; ++i) {
  370. auto p = this->ProcessorsAvailable.begin();
  371. affinity.push_back(*p);
  372. this->ProcessorsAvailable.erase(p);
  373. }
  374. properties->Affinity = std::move(affinity);
  375. }
  376. }
  377. void cmCTestMultiProcessHandler::UnlockResources(int index)
  378. {
  379. auto* properties = this->Properties[index];
  380. for (auto p : properties->Affinity) {
  381. this->ProcessorsAvailable.insert(p);
  382. }
  383. properties->Affinity.clear();
  384. for (std::string const& i : properties->ProjectResources) {
  385. this->ProjectResourcesLocked.erase(i);
  386. }
  387. if (properties->RunSerial) {
  388. this->SerialTestRunning = false;
  389. }
  390. this->RunningCount -= this->GetProcessorsUsed(index);
  391. }
  392. inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
  393. {
  394. size_t processors = static_cast<int>(this->Properties[test]->Processors);
  395. // If processors setting is set higher than the -j
  396. // setting, we default to using all of the process slots.
  397. if (processors > this->ParallelLevel) {
  398. processors = this->ParallelLevel;
  399. }
  400. // Cap tests that want affinity to the maximum affinity available.
  401. if (this->HaveAffinity && processors > this->HaveAffinity &&
  402. this->Properties[test]->WantAffinity) {
  403. processors = this->HaveAffinity;
  404. }
  405. return processors;
  406. }
  407. std::string cmCTestMultiProcessHandler::GetName(int test)
  408. {
  409. return this->Properties[test]->Name;
  410. }
  411. void cmCTestMultiProcessHandler::StartTest(int test)
  412. {
  413. if (this->JobServerClient) {
  414. // There is a job server. Request a token and queue the test to run
  415. // when a token is received. Note that if we do not get a token right
  416. // away it's possible that the system load will be higher when the
  417. // token is received and we may violate the test-load limit. However,
  418. // this is unlikely because if we do not get a token right away, some
  419. // other job that's currently running must finish before we get one.
  420. this->JobServerClient->RequestToken();
  421. this->JobServerQueuedTests.emplace_back(test);
  422. } else {
  423. // There is no job server. Start the test now.
  424. this->StartTestProcess(test);
  425. }
  426. }
  427. void cmCTestMultiProcessHandler::JobServerReceivedToken()
  428. {
  429. assert(!this->JobServerQueuedTests.empty());
  430. int test = this->JobServerQueuedTests.front();
  431. this->JobServerQueuedTests.pop_front();
  432. this->StartTestProcess(test);
  433. }
  434. void cmCTestMultiProcessHandler::StartNextTests()
  435. {
  436. // One or more events may be scheduled to call this method again.
  437. // Since this method has been called they are no longer needed.
  438. this->StartNextTestsOnIdle_.stop();
  439. this->StartNextTestsOnTimer_.stop();
  440. if (this->PendingTests.empty() || this->CheckStopTimePassed() ||
  441. (this->CheckStopOnFailure() && !this->Failed->empty())) {
  442. return;
  443. }
  444. size_t numToStart = 0;
  445. if (this->RunningCount < this->ParallelLevel) {
  446. numToStart = this->ParallelLevel - this->RunningCount;
  447. }
  448. if (numToStart == 0) {
  449. return;
  450. }
  451. // Don't start any new tests if one with the RUN_SERIAL property
  452. // is already running.
  453. if (this->SerialTestRunning) {
  454. return;
  455. }
  456. bool allTestsFailedTestLoadCheck = false;
  457. size_t minProcessorsRequired = this->ParallelLevel;
  458. std::string testWithMinProcessors;
  459. cmsys::SystemInformation info;
  460. unsigned long systemLoad = 0;
  461. size_t spareLoad = 0;
  462. if (this->TestLoad > 0) {
  463. // Activate possible wait.
  464. allTestsFailedTestLoadCheck = true;
  465. // Check for a fake load average value used in testing.
  466. if (this->FakeLoadForTesting > 0) {
  467. systemLoad = this->FakeLoadForTesting;
  468. // Drop the fake load for the next iteration to a value low enough
  469. // that the next iteration will start tests.
  470. this->FakeLoadForTesting = 1;
  471. }
  472. // If it's not set, look up the true load average.
  473. else {
  474. systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
  475. }
  476. spareLoad =
  477. (this->TestLoad > systemLoad ? this->TestLoad - systemLoad : 0);
  478. // Don't start more tests than the spare load can support.
  479. if (numToStart > spareLoad) {
  480. numToStart = spareLoad;
  481. }
  482. }
  483. // Start tests in the preferred order, each subject to readiness checks.
  484. auto ti = this->OrderedTests.begin();
  485. while (numToStart > 0 && !this->SerialTestRunning &&
  486. ti != this->OrderedTests.end()) {
  487. // Increment the test iterator now because the current list
  488. // entry may be deleted below.
  489. auto cti = ti++;
  490. int test = *cti;
  491. // We can only start a RUN_SERIAL test if no other tests are also
  492. // running.
  493. if (this->Properties[test]->RunSerial && this->RunningCount > 0) {
  494. continue;
  495. }
  496. // Exclude tests that depend on unfinished tests.
  497. if (!this->PendingTests[test].Depends.empty()) {
  498. continue;
  499. }
  500. size_t processors = this->GetProcessorsUsed(test);
  501. if (this->TestLoad > 0) {
  502. // Exclude tests that are too big to fit in the spare load.
  503. if (processors > spareLoad) {
  504. // Keep track of the smallest excluded test to report in message below.
  505. if (processors <= minProcessorsRequired) {
  506. minProcessorsRequired = processors;
  507. testWithMinProcessors = this->GetName(test);
  508. }
  509. continue;
  510. }
  511. // We found a test that fits in the spare load.
  512. allTestsFailedTestLoadCheck = false;
  513. cmCTestLog(this->CTest, DEBUG,
  514. "OK to run "
  515. << this->GetName(test) << ", it requires " << processors
  516. << " procs & system load is: " << systemLoad << std::endl);
  517. }
  518. // Exclude tests that are too big to fit in the concurrency limit.
  519. if (processors > numToStart) {
  520. continue;
  521. }
  522. // Exclude tests that depend on currently-locked project resources.
  523. for (std::string const& i : this->Properties[test]->ProjectResources) {
  524. if (cm::contains(this->ProjectResourcesLocked, i)) {
  525. continue;
  526. }
  527. }
  528. // Allocate system resources needed by this test.
  529. if (!this->AllocateResources(test)) {
  530. continue;
  531. }
  532. // Lock resources needed by this test.
  533. this->LockResources(test);
  534. // The test is ready to run.
  535. numToStart -= processors;
  536. this->OrderedTests.erase(cti);
  537. this->PendingTests.erase(test);
  538. this->StartTest(test);
  539. }
  540. if (allTestsFailedTestLoadCheck) {
  541. // Find out whether there are any non RUN_SERIAL tests left, so that the
  542. // correct warning may be displayed.
  543. bool onlyRunSerialTestsLeft = true;
  544. for (auto const& t : this->PendingTests) {
  545. if (!this->Properties[t.first]->RunSerial) {
  546. onlyRunSerialTestsLeft = false;
  547. }
  548. }
  549. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "***** WAITING, ");
  550. if (this->SerialTestRunning) {
  551. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  552. "Waiting for RUN_SERIAL test to finish.");
  553. } else if (onlyRunSerialTestsLeft) {
  554. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  555. "Only RUN_SERIAL tests remain, awaiting available slot.");
  556. } else if (!testWithMinProcessors.empty()) {
  557. /* clang-format off */
  558. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  559. "System Load: " << systemLoad << ", "
  560. "Max Allowed Load: " << this->TestLoad << ", "
  561. "Smallest test " << testWithMinProcessors <<
  562. " requires " << minProcessorsRequired);
  563. /* clang-format on */
  564. } else {
  565. /* clang-format off */
  566. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  567. "System Load: " << systemLoad << ", "
  568. "Max Allowed Load: " << this->TestLoad);
  569. /* clang-format on */
  570. }
  571. cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "*****" << std::endl);
  572. // Try again later when the load might be lower.
  573. this->StartNextTestsOnTimer();
  574. }
  575. }
  576. void cmCTestMultiProcessHandler::StartNextTestsOnIdle()
  577. {
  578. // Start more tests on the next loop iteration.
  579. this->StartNextTestsOnIdle_.start([](uv_idle_t* idle) {
  580. uv_idle_stop(idle);
  581. auto* self = static_cast<cmCTestMultiProcessHandler*>(idle->data);
  582. self->StartNextTests();
  583. });
  584. }
  585. void cmCTestMultiProcessHandler::StartNextTestsOnTimer()
  586. {
  587. // Wait between 1 and 5 seconds before trying again.
  588. unsigned int const milliseconds = this->FakeLoadForTesting
  589. ? 10
  590. : (cmSystemTools::RandomSeed() % 5 + 1) * 1000;
  591. this->StartNextTestsOnTimer_.start(
  592. [](uv_timer_t* timer) {
  593. uv_timer_stop(timer);
  594. auto* self = static_cast<cmCTestMultiProcessHandler*>(timer->data);
  595. self->StartNextTests();
  596. },
  597. milliseconds, 0);
  598. }
  599. void cmCTestMultiProcessHandler::FinishTestProcess(
  600. std::unique_ptr<cmCTestRunTest> runner, bool started)
  601. {
  602. this->Completed++;
  603. int test = runner->GetIndex();
  604. auto* properties = runner->GetTestProperties();
  605. cmCTestRunTest::EndTestResult testResult =
  606. runner->EndTest(this->Completed, this->Total, started);
  607. if (testResult.StopTimePassed) {
  608. this->SetStopTimePassed();
  609. }
  610. if (started) {
  611. if (!this->StopTimePassed &&
  612. cmCTestRunTest::StartAgain(std::move(runner), this->Completed)) {
  613. this->Completed--; // remove the completed test because run again
  614. return;
  615. }
  616. }
  617. if (testResult.Passed) {
  618. this->Passed->push_back(properties->Name);
  619. } else if (!properties->Disabled) {
  620. this->Failed->push_back(properties->Name);
  621. }
  622. for (auto& t : this->PendingTests) {
  623. t.second.Depends.erase(test);
  624. }
  625. this->WriteCheckpoint(test);
  626. this->DeallocateResources(test);
  627. this->UnlockResources(test);
  628. runner.reset();
  629. if (this->JobServerClient) {
  630. this->JobServerClient->ReleaseToken();
  631. }
  632. this->StartNextTestsOnIdle();
  633. }
  634. void cmCTestMultiProcessHandler::UpdateCostData()
  635. {
  636. std::string fname = this->CTest->GetCostDataFile();
  637. std::string tmpout = fname + ".tmp";
  638. cmsys::ofstream fout;
  639. fout.open(tmpout.c_str());
  640. PropertiesMap temp = this->Properties;
  641. if (cmSystemTools::FileExists(fname)) {
  642. cmsys::ifstream fin;
  643. fin.open(fname.c_str());
  644. std::string line;
  645. while (std::getline(fin, line)) {
  646. if (line == "---") {
  647. break;
  648. }
  649. std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
  650. // Format: <name> <previous_runs> <avg_cost>
  651. if (parts.size() < 3) {
  652. break;
  653. }
  654. std::string name = parts[0];
  655. int prev = atoi(parts[1].c_str());
  656. float cost = static_cast<float>(atof(parts[2].c_str()));
  657. int index = this->SearchByName(name);
  658. if (index == -1) {
  659. // This test is not in memory. We just rewrite the entry
  660. fout << name << " " << prev << " " << cost << "\n";
  661. } else {
  662. // Update with our new average cost
  663. fout << name << " " << this->Properties[index]->PreviousRuns << " "
  664. << this->Properties[index]->Cost << "\n";
  665. temp.erase(index);
  666. }
  667. }
  668. fin.close();
  669. cmSystemTools::RemoveFile(fname);
  670. }
  671. // Add all tests not previously listed in the file
  672. for (auto const& i : temp) {
  673. fout << i.second->Name << " " << i.second->PreviousRuns << " "
  674. << i.second->Cost << "\n";
  675. }
  676. // Write list of failed tests
  677. fout << "---\n";
  678. for (std::string const& f : *this->Failed) {
  679. fout << f << "\n";
  680. }
  681. fout.close();
  682. cmSystemTools::RenameFile(tmpout, fname);
  683. }
  684. void cmCTestMultiProcessHandler::ReadCostData()
  685. {
  686. std::string fname = this->CTest->GetCostDataFile();
  687. if (cmSystemTools::FileExists(fname, true)) {
  688. cmsys::ifstream fin;
  689. fin.open(fname.c_str());
  690. std::string line;
  691. while (std::getline(fin, line)) {
  692. if (line == "---") {
  693. break;
  694. }
  695. std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
  696. // Probably an older version of the file, will be fixed next run
  697. if (parts.size() < 3) {
  698. fin.close();
  699. return;
  700. }
  701. std::string name = parts[0];
  702. int prev = atoi(parts[1].c_str());
  703. float cost = static_cast<float>(atof(parts[2].c_str()));
  704. int index = this->SearchByName(name);
  705. if (index == -1) {
  706. continue;
  707. }
  708. this->Properties[index]->PreviousRuns = prev;
  709. // When not running in parallel mode, don't use cost data
  710. if (this->ParallelLevel > 1 && this->Properties[index] &&
  711. this->Properties[index]->Cost == 0) {
  712. this->Properties[index]->Cost = cost;
  713. }
  714. }
  715. // Next part of the file is the failed tests
  716. while (std::getline(fin, line)) {
  717. if (!line.empty()) {
  718. this->LastTestsFailed.push_back(line);
  719. }
  720. }
  721. fin.close();
  722. }
  723. }
  724. int cmCTestMultiProcessHandler::SearchByName(std::string const& name)
  725. {
  726. int index = -1;
  727. for (auto const& p : this->Properties) {
  728. if (p.second->Name == name) {
  729. index = p.first;
  730. }
  731. }
  732. return index;
  733. }
  734. void cmCTestMultiProcessHandler::CreateTestCostList()
  735. {
  736. if (this->ParallelLevel > 1) {
  737. this->CreateParallelTestCostList();
  738. } else {
  739. this->CreateSerialTestCostList();
  740. }
  741. }
  742. void cmCTestMultiProcessHandler::CreateParallelTestCostList()
  743. {
  744. TestSet alreadyOrderedTests;
  745. std::list<TestSet> priorityStack;
  746. priorityStack.emplace_back();
  747. TestSet& topLevel = priorityStack.back();
  748. // In parallel test runs add previously failed tests to the front
  749. // of the cost list and queue other tests for further sorting
  750. for (auto const& t : this->PendingTests) {
  751. if (cm::contains(this->LastTestsFailed, this->Properties[t.first]->Name)) {
  752. // If the test failed last time, it should be run first.
  753. this->OrderedTests.push_back(t.first);
  754. alreadyOrderedTests.insert(t.first);
  755. } else {
  756. topLevel.insert(t.first);
  757. }
  758. }
  759. // In parallel test runs repeatedly move dependencies of the tests on
  760. // the current dependency level to the next level until no
  761. // further dependencies exist.
  762. while (!priorityStack.back().empty()) {
  763. TestSet& previousSet = priorityStack.back();
  764. priorityStack.emplace_back();
  765. TestSet& currentSet = priorityStack.back();
  766. for (auto const& i : previousSet) {
  767. TestSet const& dependencies = this->PendingTests[i].Depends;
  768. currentSet.insert(dependencies.begin(), dependencies.end());
  769. }
  770. for (auto const& i : currentSet) {
  771. previousSet.erase(i);
  772. }
  773. }
  774. // Remove the empty dependency level
  775. priorityStack.pop_back();
  776. // Reverse iterate over the different dependency levels (deepest first).
  777. // Sort tests within each level by COST and append them to the cost list.
  778. for (TestSet const& currentSet : cmReverseRange(priorityStack)) {
  779. TestList sortedCopy;
  780. cm::append(sortedCopy, currentSet);
  781. std::stable_sort(sortedCopy.begin(), sortedCopy.end(),
  782. TestComparator(this));
  783. for (auto const& j : sortedCopy) {
  784. if (!cm::contains(alreadyOrderedTests, j)) {
  785. this->OrderedTests.push_back(j);
  786. alreadyOrderedTests.insert(j);
  787. }
  788. }
  789. }
  790. }
  791. void cmCTestMultiProcessHandler::GetAllTestDependencies(int test,
  792. TestList& dependencies)
  793. {
  794. TestSet const& dependencySet = this->PendingTests[test].Depends;
  795. for (int i : dependencySet) {
  796. this->GetAllTestDependencies(i, dependencies);
  797. dependencies.push_back(i);
  798. }
  799. }
  800. void cmCTestMultiProcessHandler::CreateSerialTestCostList()
  801. {
  802. TestList presortedList;
  803. for (auto const& i : this->PendingTests) {
  804. presortedList.push_back(i.first);
  805. }
  806. std::stable_sort(presortedList.begin(), presortedList.end(),
  807. TestComparator(this));
  808. TestSet alreadyOrderedTests;
  809. for (int test : presortedList) {
  810. if (cm::contains(alreadyOrderedTests, test)) {
  811. continue;
  812. }
  813. TestList dependencies;
  814. this->GetAllTestDependencies(test, dependencies);
  815. for (int testDependency : dependencies) {
  816. if (!cm::contains(alreadyOrderedTests, testDependency)) {
  817. alreadyOrderedTests.insert(testDependency);
  818. this->OrderedTests.push_back(testDependency);
  819. }
  820. }
  821. alreadyOrderedTests.insert(test);
  822. this->OrderedTests.push_back(test);
  823. }
  824. }
  825. void cmCTestMultiProcessHandler::WriteCheckpoint(int index)
  826. {
  827. std::string fname =
  828. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  829. cmsys::ofstream fout;
  830. fout.open(fname.c_str(), std::ios::app);
  831. fout << index << "\n";
  832. fout.close();
  833. }
  834. void cmCTestMultiProcessHandler::MarkFinished()
  835. {
  836. std::string fname =
  837. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  838. cmSystemTools::RemoveFile(fname);
  839. }
  840. static Json::Value DumpToJsonArray(const std::set<std::string>& values)
  841. {
  842. Json::Value jsonArray = Json::arrayValue;
  843. for (const auto& it : values) {
  844. jsonArray.append(it);
  845. }
  846. return jsonArray;
  847. }
  848. static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
  849. {
  850. Json::Value jsonArray = Json::arrayValue;
  851. for (const auto& it : values) {
  852. jsonArray.append(it);
  853. }
  854. return jsonArray;
  855. }
  856. static Json::Value DumpRegExToJsonArray(
  857. const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
  858. {
  859. Json::Value jsonArray = Json::arrayValue;
  860. for (const auto& it : values) {
  861. jsonArray.append(it.second);
  862. }
  863. return jsonArray;
  864. }
  865. static Json::Value DumpMeasurementToJsonArray(
  866. const std::map<std::string, std::string>& values)
  867. {
  868. Json::Value jsonArray = Json::arrayValue;
  869. for (const auto& it : values) {
  870. Json::Value measurement = Json::objectValue;
  871. measurement["measurement"] = it.first;
  872. measurement["value"] = it.second;
  873. jsonArray.append(measurement);
  874. }
  875. return jsonArray;
  876. }
  877. static Json::Value DumpTimeoutAfterMatch(
  878. cmCTestTestHandler::cmCTestTestProperties& testProperties)
  879. {
  880. Json::Value timeoutAfterMatch = Json::objectValue;
  881. timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
  882. timeoutAfterMatch["regex"] =
  883. DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
  884. return timeoutAfterMatch;
  885. }
  886. static Json::Value DumpResourceGroupsToJsonArray(
  887. const std::vector<
  888. std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>>&
  889. resourceGroups)
  890. {
  891. Json::Value jsonResourceGroups = Json::arrayValue;
  892. for (auto const& it : resourceGroups) {
  893. Json::Value jsonResourceGroup = Json::objectValue;
  894. Json::Value requirements = Json::arrayValue;
  895. for (auto const& it2 : it) {
  896. Json::Value res = Json::objectValue;
  897. res[".type"] = it2.ResourceType;
  898. // res[".units"] = it2.UnitsNeeded; // Intentionally commented out
  899. res["slots"] = it2.SlotsNeeded;
  900. requirements.append(res);
  901. }
  902. jsonResourceGroup["requirements"] = requirements;
  903. jsonResourceGroups.append(jsonResourceGroup);
  904. }
  905. return jsonResourceGroups;
  906. }
  907. static Json::Value DumpCTestProperty(std::string const& name,
  908. Json::Value value)
  909. {
  910. Json::Value property = Json::objectValue;
  911. property["name"] = name;
  912. property["value"] = std::move(value);
  913. return property;
  914. }
  915. static Json::Value DumpCTestProperties(
  916. cmCTestTestHandler::cmCTestTestProperties& testProperties)
  917. {
  918. Json::Value properties = Json::arrayValue;
  919. if (!testProperties.AttachOnFail.empty()) {
  920. properties.append(DumpCTestProperty(
  921. "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
  922. }
  923. if (!testProperties.AttachedFiles.empty()) {
  924. properties.append(DumpCTestProperty(
  925. "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
  926. }
  927. if (testProperties.Cost != 0.0f) {
  928. properties.append(
  929. DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
  930. }
  931. if (!testProperties.Depends.empty()) {
  932. properties.append(
  933. DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
  934. }
  935. if (testProperties.Disabled) {
  936. properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
  937. }
  938. if (!testProperties.Environment.empty()) {
  939. properties.append(DumpCTestProperty(
  940. "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
  941. }
  942. if (!testProperties.EnvironmentModification.empty()) {
  943. properties.append(DumpCTestProperty(
  944. "ENVIRONMENT_MODIFICATION",
  945. DumpToJsonArray(testProperties.EnvironmentModification)));
  946. }
  947. if (!testProperties.ErrorRegularExpressions.empty()) {
  948. properties.append(DumpCTestProperty(
  949. "FAIL_REGULAR_EXPRESSION",
  950. DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
  951. }
  952. if (!testProperties.SkipRegularExpressions.empty()) {
  953. properties.append(DumpCTestProperty(
  954. "SKIP_REGULAR_EXPRESSION",
  955. DumpRegExToJsonArray(testProperties.SkipRegularExpressions)));
  956. }
  957. if (!testProperties.FixturesCleanup.empty()) {
  958. properties.append(DumpCTestProperty(
  959. "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
  960. }
  961. if (!testProperties.FixturesRequired.empty()) {
  962. properties.append(DumpCTestProperty(
  963. "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
  964. }
  965. if (!testProperties.FixturesSetup.empty()) {
  966. properties.append(DumpCTestProperty(
  967. "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
  968. }
  969. if (!testProperties.Labels.empty()) {
  970. properties.append(
  971. DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
  972. }
  973. if (!testProperties.Measurements.empty()) {
  974. properties.append(DumpCTestProperty(
  975. "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
  976. }
  977. if (!testProperties.RequiredRegularExpressions.empty()) {
  978. properties.append(DumpCTestProperty(
  979. "PASS_REGULAR_EXPRESSION",
  980. DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
  981. }
  982. if (!testProperties.ResourceGroups.empty()) {
  983. properties.append(DumpCTestProperty(
  984. "RESOURCE_GROUPS",
  985. DumpResourceGroupsToJsonArray(testProperties.ResourceGroups)));
  986. }
  987. if (testProperties.WantAffinity) {
  988. properties.append(
  989. DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
  990. }
  991. if (testProperties.Processors != 1) {
  992. properties.append(
  993. DumpCTestProperty("PROCESSORS", testProperties.Processors));
  994. }
  995. if (!testProperties.RequiredFiles.empty()) {
  996. properties.append(DumpCTestProperty(
  997. "REQUIRED_FILES", DumpToJsonArray(testProperties.RequiredFiles)));
  998. }
  999. if (!testProperties.ProjectResources.empty()) {
  1000. properties.append(DumpCTestProperty(
  1001. "RESOURCE_LOCK", DumpToJsonArray(testProperties.ProjectResources)));
  1002. }
  1003. if (testProperties.RunSerial) {
  1004. properties.append(
  1005. DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
  1006. }
  1007. if (testProperties.SkipReturnCode != -1) {
  1008. properties.append(
  1009. DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
  1010. }
  1011. if (testProperties.Timeout) {
  1012. properties.append(
  1013. DumpCTestProperty("TIMEOUT", testProperties.Timeout->count()));
  1014. }
  1015. if (!testProperties.TimeoutRegularExpressions.empty()) {
  1016. properties.append(DumpCTestProperty(
  1017. "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
  1018. }
  1019. if (testProperties.WillFail) {
  1020. properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
  1021. }
  1022. if (!testProperties.Directory.empty()) {
  1023. properties.append(
  1024. DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
  1025. }
  1026. return properties;
  1027. }
  1028. class BacktraceData
  1029. {
  1030. std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
  1031. std::unordered_map<std::string, Json::ArrayIndex> FileMap;
  1032. std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
  1033. Json::Value Commands = Json::arrayValue;
  1034. Json::Value Files = Json::arrayValue;
  1035. Json::Value Nodes = Json::arrayValue;
  1036. Json::ArrayIndex AddCommand(std::string const& command)
  1037. {
  1038. auto i = this->CommandMap.find(command);
  1039. if (i == this->CommandMap.end()) {
  1040. i = this->CommandMap.emplace(command, this->Commands.size()).first;
  1041. this->Commands.append(command);
  1042. }
  1043. return i->second;
  1044. }
  1045. Json::ArrayIndex AddFile(std::string const& file)
  1046. {
  1047. auto i = this->FileMap.find(file);
  1048. if (i == this->FileMap.end()) {
  1049. i = this->FileMap.emplace(file, this->Files.size()).first;
  1050. this->Files.append(file);
  1051. }
  1052. return i->second;
  1053. }
  1054. public:
  1055. bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
  1056. Json::Value Dump();
  1057. };
  1058. bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
  1059. {
  1060. if (bt.Empty()) {
  1061. return false;
  1062. }
  1063. cmListFileContext const* top = &bt.Top();
  1064. auto found = this->NodeMap.find(top);
  1065. if (found != this->NodeMap.end()) {
  1066. index = found->second;
  1067. return true;
  1068. }
  1069. Json::Value entry = Json::objectValue;
  1070. entry["file"] = this->AddFile(top->FilePath);
  1071. if (top->Line) {
  1072. entry["line"] = static_cast<int>(top->Line);
  1073. }
  1074. if (!top->Name.empty()) {
  1075. entry["command"] = this->AddCommand(top->Name);
  1076. }
  1077. Json::ArrayIndex parent;
  1078. if (this->Add(bt.Pop(), parent)) {
  1079. entry["parent"] = parent;
  1080. }
  1081. index = this->NodeMap[top] = this->Nodes.size();
  1082. this->Nodes.append(std::move(entry)); // NOLINT(*)
  1083. return true;
  1084. }
  1085. Json::Value BacktraceData::Dump()
  1086. {
  1087. Json::Value backtraceGraph;
  1088. this->CommandMap.clear();
  1089. this->FileMap.clear();
  1090. this->NodeMap.clear();
  1091. backtraceGraph["commands"] = std::move(this->Commands);
  1092. backtraceGraph["files"] = std::move(this->Files);
  1093. backtraceGraph["nodes"] = std::move(this->Nodes);
  1094. return backtraceGraph;
  1095. }
  1096. static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
  1097. cmListFileBacktrace const& bt)
  1098. {
  1099. Json::ArrayIndex backtrace;
  1100. if (backtraceGraph.Add(bt, backtrace)) {
  1101. object["backtrace"] = backtrace;
  1102. }
  1103. }
  1104. static Json::Value DumpCTestInfo(
  1105. cmCTestRunTest& testRun,
  1106. cmCTestTestHandler::cmCTestTestProperties& testProperties,
  1107. BacktraceData& backtraceGraph)
  1108. {
  1109. Json::Value testInfo = Json::objectValue;
  1110. // test name should always be present
  1111. testInfo["name"] = testProperties.Name;
  1112. std::string const& config = testRun.GetCTest()->GetConfigType();
  1113. if (!config.empty()) {
  1114. testInfo["config"] = config;
  1115. }
  1116. std::string const& command = testRun.GetActualCommand();
  1117. if (!command.empty()) {
  1118. std::vector<std::string> commandAndArgs;
  1119. commandAndArgs.push_back(command);
  1120. const std::vector<std::string>& args = testRun.GetArguments();
  1121. if (!args.empty()) {
  1122. commandAndArgs.reserve(args.size() + 1);
  1123. cm::append(commandAndArgs, args);
  1124. }
  1125. testInfo["command"] = DumpToJsonArray(commandAndArgs);
  1126. }
  1127. Json::Value properties = DumpCTestProperties(testProperties);
  1128. if (!properties.empty()) {
  1129. testInfo["properties"] = properties;
  1130. }
  1131. if (!testProperties.Backtrace.Empty()) {
  1132. AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
  1133. }
  1134. return testInfo;
  1135. }
  1136. static Json::Value DumpVersion(int major, int minor)
  1137. {
  1138. Json::Value version = Json::objectValue;
  1139. version["major"] = major;
  1140. version["minor"] = minor;
  1141. return version;
  1142. }
  1143. void cmCTestMultiProcessHandler::PrintOutputAsJson()
  1144. {
  1145. this->TestHandler->SetMaxIndex(this->FindMaxIndex());
  1146. Json::Value result = Json::objectValue;
  1147. result["kind"] = "ctestInfo";
  1148. result["version"] = DumpVersion(1, 0);
  1149. BacktraceData backtraceGraph;
  1150. Json::Value tests = Json::arrayValue;
  1151. for (auto& it : this->Properties) {
  1152. cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
  1153. // Don't worry if this fails, we are only showing the test list, not
  1154. // running the tests
  1155. cmWorkingDirectory workdir(p.Directory);
  1156. cmCTestRunTest testRun(*this, p.Index);
  1157. testRun.ComputeArguments();
  1158. // Skip tests not available in this configuration.
  1159. if (p.Args.size() >= 2 && p.Args[1] == "NOT_AVAILABLE") {
  1160. continue;
  1161. }
  1162. Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
  1163. tests.append(testInfo);
  1164. }
  1165. result["backtraceGraph"] = backtraceGraph.Dump();
  1166. result["tests"] = std::move(tests);
  1167. Json::StreamWriterBuilder builder;
  1168. builder["indentation"] = " ";
  1169. std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
  1170. jout->write(result, &std::cout);
  1171. }
  1172. // For ShowOnly mode
  1173. void cmCTestMultiProcessHandler::PrintTestList()
  1174. {
  1175. if (this->CTest->GetOutputAsJson()) {
  1176. this->PrintOutputAsJson();
  1177. return;
  1178. }
  1179. this->TestHandler->SetMaxIndex(this->FindMaxIndex());
  1180. for (auto& it : this->Properties) {
  1181. cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
  1182. // Don't worry if this fails, we are only showing the test list, not
  1183. // running the tests
  1184. cmWorkingDirectory workdir(p.Directory);
  1185. cmCTestRunTest testRun(*this, p.Index);
  1186. testRun.ComputeArguments(); // logs the command in verbose mode
  1187. if (!p.Labels.empty()) // print the labels
  1188. {
  1189. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  1190. "Labels:", this->Quiet);
  1191. }
  1192. for (std::string const& label : p.Labels) {
  1193. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, " " << label,
  1194. this->Quiet);
  1195. }
  1196. if (!p.Labels.empty()) // print the labels
  1197. {
  1198. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl,
  1199. this->Quiet);
  1200. }
  1201. if (this->TestHandler->MemCheck) {
  1202. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Memory Check",
  1203. this->Quiet);
  1204. } else {
  1205. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Test", this->Quiet);
  1206. }
  1207. std::ostringstream indexStr;
  1208. indexStr << " #" << p.Index << ":";
  1209. cmCTestOptionalLog(
  1210. this->CTest, HANDLER_OUTPUT,
  1211. std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex()))
  1212. << indexStr.str(),
  1213. this->Quiet);
  1214. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << p.Name,
  1215. this->Quiet);
  1216. if (p.Disabled) {
  1217. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " (Disabled)",
  1218. this->Quiet);
  1219. }
  1220. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, std::endl, this->Quiet);
  1221. }
  1222. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
  1223. std::endl
  1224. << "Total Tests: " << this->Total << std::endl,
  1225. this->Quiet);
  1226. }
  1227. void cmCTestMultiProcessHandler::PrintLabels()
  1228. {
  1229. std::set<std::string> allLabels;
  1230. for (auto& it : this->Properties) {
  1231. cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
  1232. allLabels.insert(p.Labels.begin(), p.Labels.end());
  1233. }
  1234. if (!allLabels.empty()) {
  1235. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "All Labels:" << std::endl,
  1236. this->Quiet);
  1237. } else {
  1238. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
  1239. "No Labels Exist" << std::endl, this->Quiet);
  1240. }
  1241. for (std::string const& label : allLabels) {
  1242. cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << label << std::endl,
  1243. this->Quiet);
  1244. }
  1245. }
  1246. void cmCTestMultiProcessHandler::CheckResume()
  1247. {
  1248. std::string fname =
  1249. this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
  1250. if (this->CTest->GetFailover()) {
  1251. if (cmSystemTools::FileExists(fname, true)) {
  1252. *this->TestHandler->LogFile
  1253. << "Resuming previously interrupted test set" << std::endl
  1254. << "----------------------------------------------------------"
  1255. << std::endl;
  1256. cmsys::ifstream fin;
  1257. fin.open(fname.c_str());
  1258. std::string line;
  1259. while (std::getline(fin, line)) {
  1260. int index = atoi(line.c_str());
  1261. this->RemoveTest(index);
  1262. }
  1263. fin.close();
  1264. }
  1265. } else if (cmSystemTools::FileExists(fname, true)) {
  1266. cmSystemTools::RemoveFile(fname);
  1267. }
  1268. }
  1269. void cmCTestMultiProcessHandler::RemoveTest(int index)
  1270. {
  1271. this->OrderedTests.erase(
  1272. std::find(this->OrderedTests.begin(), this->OrderedTests.end(), index));
  1273. this->PendingTests.erase(index);
  1274. this->Properties.erase(index);
  1275. this->Completed++;
  1276. }
  1277. int cmCTestMultiProcessHandler::FindMaxIndex()
  1278. {
  1279. int max = 0;
  1280. for (auto const& i : this->PendingTests) {
  1281. if (i.first > max) {
  1282. max = i.first;
  1283. }
  1284. }
  1285. return max;
  1286. }
  1287. // Returns true if no cycles exist in the dependency graph
  1288. bool cmCTestMultiProcessHandler::CheckCycles()
  1289. {
  1290. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  1291. "Checking test dependency graph..." << std::endl,
  1292. this->Quiet);
  1293. for (auto const& it : this->PendingTests) {
  1294. // DFS from each element to itself
  1295. int root = it.first;
  1296. std::set<int> visited;
  1297. std::stack<int> s;
  1298. s.push(root);
  1299. while (!s.empty()) {
  1300. int test = s.top();
  1301. s.pop();
  1302. if (visited.insert(test).second) {
  1303. for (auto const& d : this->PendingTests[test].Depends) {
  1304. if (d == root) {
  1305. // cycle exists
  1306. cmCTestLog(
  1307. this->CTest, ERROR_MESSAGE,
  1308. "Error: a cycle exists in the test dependency graph "
  1309. "for the test \""
  1310. << this->Properties[root]->Name
  1311. << "\".\nPlease fix the cycle and run ctest again.\n");
  1312. return false;
  1313. }
  1314. s.push(d);
  1315. }
  1316. }
  1317. }
  1318. }
  1319. cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
  1320. "Checking test dependency graph end" << std::endl,
  1321. this->Quiet);
  1322. return true;
  1323. }
  1324. bool cmCTestMultiProcessHandler::CheckGeneratedResourceSpec()
  1325. {
  1326. for (auto& test : this->Properties) {
  1327. if (!test.second->GeneratedResourceSpecFile.empty()) {
  1328. if (this->ResourceSpecSetupTest) {
  1329. cmCTestLog(
  1330. this->CTest, ERROR_MESSAGE,
  1331. "Only one test may define the GENERATED_RESOURCE_SPEC_FILE property"
  1332. << std::endl);
  1333. return false;
  1334. }
  1335. if (test.second->FixturesSetup.size() != 1) {
  1336. cmCTestLog(this->CTest, ERROR_MESSAGE,
  1337. "Test that defines GENERATED_RESOURCE_SPEC_FILE must have "
  1338. "exactly one FIXTURES_SETUP"
  1339. << std::endl);
  1340. return false;
  1341. }
  1342. if (!cmSystemTools::FileIsFullPath(
  1343. test.second->GeneratedResourceSpecFile)) {
  1344. cmCTestLog(this->CTest, ERROR_MESSAGE,
  1345. "GENERATED_RESOURCE_SPEC_FILE must be an absolute path"
  1346. << std::endl);
  1347. return false;
  1348. }
  1349. this->ResourceSpecSetupTest = test.first;
  1350. this->ResourceSpecSetupFixture = *test.second->FixturesSetup.begin();
  1351. }
  1352. }
  1353. if (!this->ResourceSpecSetupFixture.empty()) {
  1354. for (auto& test : this->Properties) {
  1355. if (!test.second->ResourceGroups.empty() &&
  1356. !test.second->FixturesRequired.count(
  1357. this->ResourceSpecSetupFixture)) {
  1358. cmCTestLog(this->CTest, ERROR_MESSAGE,
  1359. "All tests that have RESOURCE_GROUPS must include the "
  1360. "resource spec generator fixture in their FIXTURES_REQUIRED"
  1361. << std::endl);
  1362. return false;
  1363. }
  1364. }
  1365. }
  1366. if (!this->ResourceSpecFile.empty()) {
  1367. if (this->ResourceSpecSetupTest) {
  1368. cmCTestLog(this->CTest, ERROR_MESSAGE,
  1369. "GENERATED_RESOURCE_SPEC_FILE test property cannot be used "
  1370. "in conjunction with ResourceSpecFile option"
  1371. << std::endl);
  1372. return false;
  1373. }
  1374. std::string error;
  1375. if (!this->InitResourceAllocator(error)) {
  1376. cmCTestLog(this->CTest, ERROR_MESSAGE, error << std::endl);
  1377. return false;
  1378. }
  1379. }
  1380. return true;
  1381. }
  1382. bool cmCTestMultiProcessHandler::InitResourceAllocator(std::string& error)
  1383. {
  1384. if (!this->ResourceSpec.ReadFromJSONFile(this->ResourceSpecFile)) {
  1385. error = cmStrCat("Could not read/parse resource spec file ",
  1386. this->ResourceSpecFile, ": ",
  1387. this->ResourceSpec.parseState.GetErrorMessage());
  1388. return false;
  1389. }
  1390. this->UseResourceSpec = true;
  1391. this->ResourceAllocator.InitializeFromResourceSpec(this->ResourceSpec);
  1392. return true;
  1393. }