12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574 |
- /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
- file Copyright.txt or https://cmake.org/licensing for details. */
- #include "cmCTestMultiProcessHandler.h"
- #include <algorithm>
- #include <cassert>
- #include <chrono>
- #include <cmath>
- #include <cstddef> // IWYU pragma: keep
- #include <cstdlib>
- #include <cstring>
- #include <iomanip>
- #include <iostream>
- #include <list>
- #include <sstream>
- #include <stack>
- #include <unordered_map>
- #include <utility>
- #include <vector>
- #include <cm/memory>
- #include <cm/optional>
- #include <cmext/algorithm>
- #include <cm3p/json/value.h>
- #include <cm3p/json/writer.h>
- #include <cm3p/uv.h>
- #include "cmsys/FStream.hxx"
- #include "cmsys/SystemInformation.hxx"
- #include "cmAffinity.h"
- #include "cmCTest.h"
- #include "cmCTestBinPacker.h"
- #include "cmCTestRunTest.h"
- #include "cmCTestTestHandler.h"
- #include "cmDuration.h"
- #include "cmJSONState.h"
- #include "cmListFileCache.h"
- #include "cmRange.h"
- #include "cmStringAlgorithms.h"
- #include "cmSystemTools.h"
- #include "cmUVJobServerClient.h"
- #include "cmWorkingDirectory.h"
- namespace cmsys {
- class RegularExpression;
- }
- class TestComparator
- {
- public:
- TestComparator(cmCTestMultiProcessHandler* handler)
- : Handler(handler)
- {
- }
- // Sorts tests in descending order of cost
- bool operator()(int index1, int index2) const
- {
- return this->Handler->Properties[index1]->Cost >
- this->Handler->Properties[index2]->Cost;
- }
- private:
- cmCTestMultiProcessHandler* Handler;
- };
- cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
- {
- this->ParallelLevel = 1;
- this->TestLoad = 0;
- this->FakeLoadForTesting = 0;
- this->Completed = 0;
- this->RunningCount = 0;
- this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable();
- this->HaveAffinity = this->ProcessorsAvailable.size();
- this->HasCycles = false;
- this->HasInvalidGeneratedResourceSpec = false;
- this->SerialTestRunning = false;
- }
- cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
- // Set the tests
- void cmCTestMultiProcessHandler::SetTests(TestMap tests,
- PropertiesMap properties)
- {
- this->PendingTests = std::move(tests);
- this->Properties = std::move(properties);
- this->Total = this->PendingTests.size();
- if (!this->CTest->GetShowOnly()) {
- this->ReadCostData();
- this->HasCycles = !this->CheckCycles();
- this->HasInvalidGeneratedResourceSpec =
- !this->CheckGeneratedResourceSpec();
- if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
- return;
- }
- this->CreateTestCostList();
- }
- }
- // Set the max number of tests that can be run at the same time.
- void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
- {
- this->ParallelLevel = level < 1 ? 1 : level;
- }
- void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
- {
- this->TestLoad = load;
- std::string fake_load_value;
- if (cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING",
- fake_load_value)) {
- if (!cmStrToULong(fake_load_value, &this->FakeLoadForTesting)) {
- cmSystemTools::Error("Failed to parse fake load value: " +
- fake_load_value);
- }
- }
- }
- bool cmCTestMultiProcessHandler::Complete()
- {
- return this->Completed == this->Total;
- }
- void cmCTestMultiProcessHandler::InitializeLoop()
- {
- this->Loop.init();
- this->StartNextTestsOnIdle_.init(*this->Loop, this);
- this->StartNextTestsOnTimer_.init(*this->Loop, this);
- this->JobServerClient = cmUVJobServerClient::Connect(
- *this->Loop, /*onToken=*/[this]() { this->JobServerReceivedToken(); },
- /*onDisconnect=*/nullptr);
- if (this->JobServerClient) {
- cmCTestLog(this->CTest, OUTPUT,
- "Connected to MAKE jobserver" << std::endl);
- }
- }
- void cmCTestMultiProcessHandler::FinalizeLoop()
- {
- this->JobServerClient.reset();
- this->StartNextTestsOnTimer_.reset();
- this->StartNextTestsOnIdle_.reset();
- this->Loop.reset();
- }
- void cmCTestMultiProcessHandler::RunTests()
- {
- this->CheckResume();
- if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
- return;
- }
- this->TestHandler->SetMaxIndex(this->FindMaxIndex());
- this->InitializeLoop();
- this->StartNextTestsOnIdle();
- uv_run(this->Loop, UV_RUN_DEFAULT);
- this->FinalizeLoop();
- if (!this->StopTimePassed && !this->CheckStopOnFailure()) {
- assert(this->Complete());
- assert(this->PendingTests.empty());
- }
- assert(this->AllResourcesAvailable());
- this->MarkFinished();
- this->UpdateCostData();
- }
- void cmCTestMultiProcessHandler::StartTestProcess(int test)
- {
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "test " << test << "\n", this->Quiet);
- auto testRun = cm::make_unique<cmCTestRunTest>(*this, test);
- if (this->RepeatMode != cmCTest::Repeat::Never) {
- testRun->SetRepeatMode(this->RepeatMode);
- testRun->SetNumberOfRuns(this->RepeatCount);
- }
- if (this->UseResourceSpec) {
- testRun->SetUseAllocatedResources(true);
- testRun->SetAllocatedResources(this->AllocatedResources[test]);
- }
- // Find any failed dependencies for this test. We assume the more common
- // scenario has no failed tests, so make it the outer loop.
- for (std::string const& f : *this->Failed) {
- if (cm::contains(this->Properties[test]->RequireSuccessDepends, f)) {
- testRun->AddFailedDependency(f);
- }
- }
- if (!this->ResourceAvailabilityErrors[test].empty()) {
- std::ostringstream e;
- e << "Insufficient resources for test " << this->Properties[test]->Name
- << ":\n\n";
- for (auto const& it : this->ResourceAvailabilityErrors[test]) {
- switch (it.second) {
- case ResourceAvailabilityError::NoResourceType:
- e << " Test requested resources of type '" << it.first
- << "' which does not exist\n";
- break;
- case ResourceAvailabilityError::InsufficientResources:
- e << " Test requested resources of type '" << it.first
- << "' in the following amounts:\n";
- for (auto const& group : this->Properties[test]->ResourceGroups) {
- for (auto const& requirement : group) {
- if (requirement.ResourceType == it.first) {
- e << " " << requirement.SlotsNeeded
- << (requirement.SlotsNeeded == 1 ? " slot\n" : " slots\n");
- }
- }
- }
- e << " but only the following units were available:\n";
- for (auto const& res :
- this->ResourceAllocator.GetResources().at(it.first)) {
- e << " '" << res.first << "': " << res.second.Total
- << (res.second.Total == 1 ? " slot\n" : " slots\n");
- }
- break;
- }
- e << "\n";
- }
- e << "Resource spec file:\n\n " << this->ResourceSpecFile;
- cmCTestRunTest::StartFailure(std::move(testRun), this->Total, e.str(),
- "Insufficient resources");
- return;
- }
- cmWorkingDirectory workdir(this->Properties[test]->Directory);
- if (workdir.Failed()) {
- cmCTestRunTest::StartFailure(std::move(testRun), this->Total,
- "Failed to change working directory to " +
- this->Properties[test]->Directory + " : " +
- std::strerror(workdir.GetLastResult()),
- "Failed to change working directory");
- return;
- }
- // Ownership of 'testRun' has moved to another structure.
- // When the test finishes, FinishTestProcess will be called.
- cmCTestRunTest::StartTest(std::move(testRun), this->Completed, this->Total);
- }
- bool cmCTestMultiProcessHandler::AllocateResources(int index)
- {
- if (!this->UseResourceSpec) {
- return true;
- }
- // If the test needs unavailable resources then do not allocate anything
- // because it will never run. We will issue the recorded errors instead.
- if (!this->ResourceAvailabilityErrors[index].empty()) {
- return true;
- }
- std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
- if (!this->TryAllocateResources(index, allocations)) {
- return false;
- }
- auto& allocatedResources = this->AllocatedResources[index];
- allocatedResources.resize(this->Properties[index]->ResourceGroups.size());
- for (auto const& it : allocations) {
- for (auto const& alloc : it.second) {
- bool result = this->ResourceAllocator.AllocateResource(
- it.first, alloc.Id, alloc.SlotsNeeded);
- (void)result;
- assert(result);
- allocatedResources[alloc.ProcessIndex][it.first].push_back(
- { alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
- }
- }
- return true;
- }
- bool cmCTestMultiProcessHandler::TryAllocateResources(
- int index,
- std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations,
- std::map<std::string, ResourceAvailabilityError>* errors)
- {
- allocations.clear();
- std::size_t processIndex = 0;
- for (auto const& process : this->Properties[index]->ResourceGroups) {
- for (auto const& requirement : process) {
- for (int i = 0; i < requirement.UnitsNeeded; ++i) {
- allocations[requirement.ResourceType].push_back(
- { processIndex, requirement.SlotsNeeded, "" });
- }
- }
- ++processIndex;
- }
- bool result = true;
- auto const& availableResources = this->ResourceAllocator.GetResources();
- for (auto& it : allocations) {
- if (!availableResources.count(it.first)) {
- if (errors) {
- (*errors)[it.first] = ResourceAvailabilityError::NoResourceType;
- result = false;
- } else {
- return false;
- }
- } else if (!cmAllocateCTestResourcesRoundRobin(
- availableResources.at(it.first), it.second)) {
- if (errors) {
- (*errors)[it.first] = ResourceAvailabilityError::InsufficientResources;
- result = false;
- } else {
- return false;
- }
- }
- }
- return result;
- }
- void cmCTestMultiProcessHandler::DeallocateResources(int index)
- {
- if (!this->UseResourceSpec) {
- return;
- }
- {
- auto& allocatedResources = this->AllocatedResources[index];
- for (auto const& processAlloc : allocatedResources) {
- for (auto const& it : processAlloc) {
- auto resourceType = it.first;
- for (auto const& it2 : it.second) {
- bool success = this->ResourceAllocator.DeallocateResource(
- resourceType, it2.Id, it2.Slots);
- (void)success;
- assert(success);
- }
- }
- }
- }
- this->AllocatedResources.erase(index);
- }
- bool cmCTestMultiProcessHandler::AllResourcesAvailable()
- {
- for (auto const& it : this->ResourceAllocator.GetResources()) {
- for (auto const& it2 : it.second) {
- if (it2.second.Locked != 0) {
- return false;
- }
- }
- }
- return true;
- }
- void cmCTestMultiProcessHandler::CheckResourceAvailability()
- {
- if (this->UseResourceSpec) {
- for (auto const& t : this->PendingTests) {
- std::map<std::string, std::vector<cmCTestBinPackerAllocation>>
- allocations;
- this->TryAllocateResources(t.first, allocations,
- &this->ResourceAvailabilityErrors[t.first]);
- }
- }
- }
- bool cmCTestMultiProcessHandler::CheckStopOnFailure()
- {
- return this->CTest->GetStopOnFailure();
- }
- bool cmCTestMultiProcessHandler::CheckStopTimePassed()
- {
- if (!this->StopTimePassed) {
- std::chrono::system_clock::time_point stop_time =
- this->CTest->GetStopTime();
- if (stop_time != std::chrono::system_clock::time_point() &&
- stop_time <= std::chrono::system_clock::now()) {
- this->SetStopTimePassed();
- }
- }
- return this->StopTimePassed;
- }
- void cmCTestMultiProcessHandler::SetStopTimePassed()
- {
- if (!this->StopTimePassed) {
- cmCTestLog(this->CTest, ERROR_MESSAGE,
- "The stop time has been passed. "
- "Stopping all tests."
- << std::endl);
- this->StopTimePassed = true;
- }
- }
- void cmCTestMultiProcessHandler::LockResources(int index)
- {
- this->RunningCount += this->GetProcessorsUsed(index);
- auto* properties = this->Properties[index];
- this->ProjectResourcesLocked.insert(properties->ProjectResources.begin(),
- properties->ProjectResources.end());
- if (properties->RunSerial) {
- this->SerialTestRunning = true;
- }
- if (this->HaveAffinity && properties->WantAffinity) {
- size_t needProcessors = this->GetProcessorsUsed(index);
- assert(needProcessors <= this->ProcessorsAvailable.size());
- std::vector<size_t> affinity;
- affinity.reserve(needProcessors);
- for (size_t i = 0; i < needProcessors; ++i) {
- auto p = this->ProcessorsAvailable.begin();
- affinity.push_back(*p);
- this->ProcessorsAvailable.erase(p);
- }
- properties->Affinity = std::move(affinity);
- }
- }
- void cmCTestMultiProcessHandler::UnlockResources(int index)
- {
- auto* properties = this->Properties[index];
- for (auto p : properties->Affinity) {
- this->ProcessorsAvailable.insert(p);
- }
- properties->Affinity.clear();
- for (std::string const& i : properties->ProjectResources) {
- this->ProjectResourcesLocked.erase(i);
- }
- if (properties->RunSerial) {
- this->SerialTestRunning = false;
- }
- this->RunningCount -= this->GetProcessorsUsed(index);
- }
- inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
- {
- size_t processors = static_cast<int>(this->Properties[test]->Processors);
- // If processors setting is set higher than the -j
- // setting, we default to using all of the process slots.
- if (processors > this->ParallelLevel) {
- processors = this->ParallelLevel;
- }
- // Cap tests that want affinity to the maximum affinity available.
- if (this->HaveAffinity && processors > this->HaveAffinity &&
- this->Properties[test]->WantAffinity) {
- processors = this->HaveAffinity;
- }
- return processors;
- }
- std::string cmCTestMultiProcessHandler::GetName(int test)
- {
- return this->Properties[test]->Name;
- }
- void cmCTestMultiProcessHandler::StartTest(int test)
- {
- if (this->JobServerClient) {
- // There is a job server. Request a token and queue the test to run
- // when a token is received. Note that if we do not get a token right
- // away it's possible that the system load will be higher when the
- // token is received and we may violate the test-load limit. However,
- // this is unlikely because if we do not get a token right away, some
- // other job that's currently running must finish before we get one.
- this->JobServerClient->RequestToken();
- this->JobServerQueuedTests.emplace_back(test);
- } else {
- // There is no job server. Start the test now.
- this->StartTestProcess(test);
- }
- }
- void cmCTestMultiProcessHandler::JobServerReceivedToken()
- {
- assert(!this->JobServerQueuedTests.empty());
- int test = this->JobServerQueuedTests.front();
- this->JobServerQueuedTests.pop_front();
- this->StartTestProcess(test);
- }
- void cmCTestMultiProcessHandler::StartNextTests()
- {
- // One or more events may be scheduled to call this method again.
- // Since this method has been called they are no longer needed.
- this->StartNextTestsOnIdle_.stop();
- this->StartNextTestsOnTimer_.stop();
- if (this->PendingTests.empty() || this->CheckStopTimePassed() ||
- (this->CheckStopOnFailure() && !this->Failed->empty())) {
- return;
- }
- size_t numToStart = 0;
- if (this->RunningCount < this->ParallelLevel) {
- numToStart = this->ParallelLevel - this->RunningCount;
- }
- if (numToStart == 0) {
- return;
- }
- // Don't start any new tests if one with the RUN_SERIAL property
- // is already running.
- if (this->SerialTestRunning) {
- return;
- }
- bool allTestsFailedTestLoadCheck = false;
- size_t minProcessorsRequired = this->ParallelLevel;
- std::string testWithMinProcessors;
- cmsys::SystemInformation info;
- unsigned long systemLoad = 0;
- size_t spareLoad = 0;
- if (this->TestLoad > 0) {
- // Activate possible wait.
- allTestsFailedTestLoadCheck = true;
- // Check for a fake load average value used in testing.
- if (this->FakeLoadForTesting > 0) {
- systemLoad = this->FakeLoadForTesting;
- // Drop the fake load for the next iteration to a value low enough
- // that the next iteration will start tests.
- this->FakeLoadForTesting = 1;
- }
- // If it's not set, look up the true load average.
- else {
- systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
- }
- spareLoad =
- (this->TestLoad > systemLoad ? this->TestLoad - systemLoad : 0);
- // Don't start more tests than the spare load can support.
- if (numToStart > spareLoad) {
- numToStart = spareLoad;
- }
- }
- // Start tests in the preferred order, each subject to readiness checks.
- auto ti = this->OrderedTests.begin();
- while (numToStart > 0 && !this->SerialTestRunning &&
- ti != this->OrderedTests.end()) {
- // Increment the test iterator now because the current list
- // entry may be deleted below.
- auto cti = ti++;
- int test = *cti;
- // We can only start a RUN_SERIAL test if no other tests are also
- // running.
- if (this->Properties[test]->RunSerial && this->RunningCount > 0) {
- continue;
- }
- // Exclude tests that depend on unfinished tests.
- if (!this->PendingTests[test].Depends.empty()) {
- continue;
- }
- size_t processors = this->GetProcessorsUsed(test);
- if (this->TestLoad > 0) {
- // Exclude tests that are too big to fit in the spare load.
- if (processors > spareLoad) {
- // Keep track of the smallest excluded test to report in message below.
- if (processors <= minProcessorsRequired) {
- minProcessorsRequired = processors;
- testWithMinProcessors = this->GetName(test);
- }
- continue;
- }
- // We found a test that fits in the spare load.
- allTestsFailedTestLoadCheck = false;
- cmCTestLog(this->CTest, DEBUG,
- "OK to run "
- << this->GetName(test) << ", it requires " << processors
- << " procs & system load is: " << systemLoad << std::endl);
- }
- // Exclude tests that are too big to fit in the concurrency limit.
- if (processors > numToStart) {
- continue;
- }
- // Exclude tests that depend on currently-locked project resources.
- for (std::string const& i : this->Properties[test]->ProjectResources) {
- if (cm::contains(this->ProjectResourcesLocked, i)) {
- continue;
- }
- }
- // Allocate system resources needed by this test.
- if (!this->AllocateResources(test)) {
- continue;
- }
- // Lock resources needed by this test.
- this->LockResources(test);
- // The test is ready to run.
- numToStart -= processors;
- this->OrderedTests.erase(cti);
- this->PendingTests.erase(test);
- this->StartTest(test);
- }
- if (allTestsFailedTestLoadCheck) {
- // Find out whether there are any non RUN_SERIAL tests left, so that the
- // correct warning may be displayed.
- bool onlyRunSerialTestsLeft = true;
- for (auto const& t : this->PendingTests) {
- if (!this->Properties[t.first]->RunSerial) {
- onlyRunSerialTestsLeft = false;
- }
- }
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "***** WAITING, ");
- if (this->SerialTestRunning) {
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "Waiting for RUN_SERIAL test to finish.");
- } else if (onlyRunSerialTestsLeft) {
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "Only RUN_SERIAL tests remain, awaiting available slot.");
- } else if (!testWithMinProcessors.empty()) {
- /* clang-format off */
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "System Load: " << systemLoad << ", "
- "Max Allowed Load: " << this->TestLoad << ", "
- "Smallest test " << testWithMinProcessors <<
- " requires " << minProcessorsRequired);
- /* clang-format on */
- } else {
- /* clang-format off */
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "System Load: " << systemLoad << ", "
- "Max Allowed Load: " << this->TestLoad);
- /* clang-format on */
- }
- cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "*****" << std::endl);
- // Try again later when the load might be lower.
- this->StartNextTestsOnTimer();
- }
- }
- void cmCTestMultiProcessHandler::StartNextTestsOnIdle()
- {
- // Start more tests on the next loop iteration.
- this->StartNextTestsOnIdle_.start([](uv_idle_t* idle) {
- uv_idle_stop(idle);
- auto* self = static_cast<cmCTestMultiProcessHandler*>(idle->data);
- self->StartNextTests();
- });
- }
- void cmCTestMultiProcessHandler::StartNextTestsOnTimer()
- {
- // Wait between 1 and 5 seconds before trying again.
- unsigned int const milliseconds = this->FakeLoadForTesting
- ? 10
- : (cmSystemTools::RandomSeed() % 5 + 1) * 1000;
- this->StartNextTestsOnTimer_.start(
- [](uv_timer_t* timer) {
- uv_timer_stop(timer);
- auto* self = static_cast<cmCTestMultiProcessHandler*>(timer->data);
- self->StartNextTests();
- },
- milliseconds, 0);
- }
- void cmCTestMultiProcessHandler::FinishTestProcess(
- std::unique_ptr<cmCTestRunTest> runner, bool started)
- {
- this->Completed++;
- int test = runner->GetIndex();
- auto* properties = runner->GetTestProperties();
- cmCTestRunTest::EndTestResult testResult =
- runner->EndTest(this->Completed, this->Total, started);
- if (testResult.StopTimePassed) {
- this->SetStopTimePassed();
- }
- if (started) {
- if (!this->StopTimePassed &&
- cmCTestRunTest::StartAgain(std::move(runner), this->Completed)) {
- this->Completed--; // remove the completed test because run again
- return;
- }
- }
- if (testResult.Passed) {
- this->Passed->push_back(properties->Name);
- } else if (!properties->Disabled) {
- this->Failed->push_back(properties->Name);
- }
- for (auto& t : this->PendingTests) {
- t.second.Depends.erase(test);
- }
- this->WriteCheckpoint(test);
- this->DeallocateResources(test);
- this->UnlockResources(test);
- runner.reset();
- if (this->JobServerClient) {
- this->JobServerClient->ReleaseToken();
- }
- this->StartNextTestsOnIdle();
- }
- void cmCTestMultiProcessHandler::UpdateCostData()
- {
- std::string fname = this->CTest->GetCostDataFile();
- std::string tmpout = fname + ".tmp";
- cmsys::ofstream fout;
- fout.open(tmpout.c_str());
- PropertiesMap temp = this->Properties;
- if (cmSystemTools::FileExists(fname)) {
- cmsys::ifstream fin;
- fin.open(fname.c_str());
- std::string line;
- while (std::getline(fin, line)) {
- if (line == "---") {
- break;
- }
- std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
- // Format: <name> <previous_runs> <avg_cost>
- if (parts.size() < 3) {
- break;
- }
- std::string name = parts[0];
- int prev = atoi(parts[1].c_str());
- float cost = static_cast<float>(atof(parts[2].c_str()));
- int index = this->SearchByName(name);
- if (index == -1) {
- // This test is not in memory. We just rewrite the entry
- fout << name << " " << prev << " " << cost << "\n";
- } else {
- // Update with our new average cost
- fout << name << " " << this->Properties[index]->PreviousRuns << " "
- << this->Properties[index]->Cost << "\n";
- temp.erase(index);
- }
- }
- fin.close();
- cmSystemTools::RemoveFile(fname);
- }
- // Add all tests not previously listed in the file
- for (auto const& i : temp) {
- fout << i.second->Name << " " << i.second->PreviousRuns << " "
- << i.second->Cost << "\n";
- }
- // Write list of failed tests
- fout << "---\n";
- for (std::string const& f : *this->Failed) {
- fout << f << "\n";
- }
- fout.close();
- cmSystemTools::RenameFile(tmpout, fname);
- }
- void cmCTestMultiProcessHandler::ReadCostData()
- {
- std::string fname = this->CTest->GetCostDataFile();
- if (cmSystemTools::FileExists(fname, true)) {
- cmsys::ifstream fin;
- fin.open(fname.c_str());
- std::string line;
- while (std::getline(fin, line)) {
- if (line == "---") {
- break;
- }
- std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
- // Probably an older version of the file, will be fixed next run
- if (parts.size() < 3) {
- fin.close();
- return;
- }
- std::string name = parts[0];
- int prev = atoi(parts[1].c_str());
- float cost = static_cast<float>(atof(parts[2].c_str()));
- int index = this->SearchByName(name);
- if (index == -1) {
- continue;
- }
- this->Properties[index]->PreviousRuns = prev;
- // When not running in parallel mode, don't use cost data
- if (this->ParallelLevel > 1 && this->Properties[index] &&
- this->Properties[index]->Cost == 0) {
- this->Properties[index]->Cost = cost;
- }
- }
- // Next part of the file is the failed tests
- while (std::getline(fin, line)) {
- if (!line.empty()) {
- this->LastTestsFailed.push_back(line);
- }
- }
- fin.close();
- }
- }
- int cmCTestMultiProcessHandler::SearchByName(std::string const& name)
- {
- int index = -1;
- for (auto const& p : this->Properties) {
- if (p.second->Name == name) {
- index = p.first;
- }
- }
- return index;
- }
- void cmCTestMultiProcessHandler::CreateTestCostList()
- {
- if (this->ParallelLevel > 1) {
- this->CreateParallelTestCostList();
- } else {
- this->CreateSerialTestCostList();
- }
- }
- void cmCTestMultiProcessHandler::CreateParallelTestCostList()
- {
- TestSet alreadyOrderedTests;
- std::list<TestSet> priorityStack;
- priorityStack.emplace_back();
- TestSet& topLevel = priorityStack.back();
- // In parallel test runs add previously failed tests to the front
- // of the cost list and queue other tests for further sorting
- for (auto const& t : this->PendingTests) {
- if (cm::contains(this->LastTestsFailed, this->Properties[t.first]->Name)) {
- // If the test failed last time, it should be run first.
- this->OrderedTests.push_back(t.first);
- alreadyOrderedTests.insert(t.first);
- } else {
- topLevel.insert(t.first);
- }
- }
- // In parallel test runs repeatedly move dependencies of the tests on
- // the current dependency level to the next level until no
- // further dependencies exist.
- while (!priorityStack.back().empty()) {
- TestSet& previousSet = priorityStack.back();
- priorityStack.emplace_back();
- TestSet& currentSet = priorityStack.back();
- for (auto const& i : previousSet) {
- TestSet const& dependencies = this->PendingTests[i].Depends;
- currentSet.insert(dependencies.begin(), dependencies.end());
- }
- for (auto const& i : currentSet) {
- previousSet.erase(i);
- }
- }
- // Remove the empty dependency level
- priorityStack.pop_back();
- // Reverse iterate over the different dependency levels (deepest first).
- // Sort tests within each level by COST and append them to the cost list.
- for (TestSet const& currentSet : cmReverseRange(priorityStack)) {
- TestList sortedCopy;
- cm::append(sortedCopy, currentSet);
- std::stable_sort(sortedCopy.begin(), sortedCopy.end(),
- TestComparator(this));
- for (auto const& j : sortedCopy) {
- if (!cm::contains(alreadyOrderedTests, j)) {
- this->OrderedTests.push_back(j);
- alreadyOrderedTests.insert(j);
- }
- }
- }
- }
- void cmCTestMultiProcessHandler::GetAllTestDependencies(int test,
- TestList& dependencies)
- {
- TestSet const& dependencySet = this->PendingTests[test].Depends;
- for (int i : dependencySet) {
- this->GetAllTestDependencies(i, dependencies);
- dependencies.push_back(i);
- }
- }
- void cmCTestMultiProcessHandler::CreateSerialTestCostList()
- {
- TestList presortedList;
- for (auto const& i : this->PendingTests) {
- presortedList.push_back(i.first);
- }
- std::stable_sort(presortedList.begin(), presortedList.end(),
- TestComparator(this));
- TestSet alreadyOrderedTests;
- for (int test : presortedList) {
- if (cm::contains(alreadyOrderedTests, test)) {
- continue;
- }
- TestList dependencies;
- this->GetAllTestDependencies(test, dependencies);
- for (int testDependency : dependencies) {
- if (!cm::contains(alreadyOrderedTests, testDependency)) {
- alreadyOrderedTests.insert(testDependency);
- this->OrderedTests.push_back(testDependency);
- }
- }
- alreadyOrderedTests.insert(test);
- this->OrderedTests.push_back(test);
- }
- }
- void cmCTestMultiProcessHandler::WriteCheckpoint(int index)
- {
- std::string fname =
- this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
- cmsys::ofstream fout;
- fout.open(fname.c_str(), std::ios::app);
- fout << index << "\n";
- fout.close();
- }
- void cmCTestMultiProcessHandler::MarkFinished()
- {
- std::string fname =
- this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
- cmSystemTools::RemoveFile(fname);
- }
- static Json::Value DumpToJsonArray(const std::set<std::string>& values)
- {
- Json::Value jsonArray = Json::arrayValue;
- for (const auto& it : values) {
- jsonArray.append(it);
- }
- return jsonArray;
- }
- static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
- {
- Json::Value jsonArray = Json::arrayValue;
- for (const auto& it : values) {
- jsonArray.append(it);
- }
- return jsonArray;
- }
- static Json::Value DumpRegExToJsonArray(
- const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
- {
- Json::Value jsonArray = Json::arrayValue;
- for (const auto& it : values) {
- jsonArray.append(it.second);
- }
- return jsonArray;
- }
- static Json::Value DumpMeasurementToJsonArray(
- const std::map<std::string, std::string>& values)
- {
- Json::Value jsonArray = Json::arrayValue;
- for (const auto& it : values) {
- Json::Value measurement = Json::objectValue;
- measurement["measurement"] = it.first;
- measurement["value"] = it.second;
- jsonArray.append(measurement);
- }
- return jsonArray;
- }
- static Json::Value DumpTimeoutAfterMatch(
- cmCTestTestHandler::cmCTestTestProperties& testProperties)
- {
- Json::Value timeoutAfterMatch = Json::objectValue;
- timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
- timeoutAfterMatch["regex"] =
- DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
- return timeoutAfterMatch;
- }
- static Json::Value DumpResourceGroupsToJsonArray(
- const std::vector<
- std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>>&
- resourceGroups)
- {
- Json::Value jsonResourceGroups = Json::arrayValue;
- for (auto const& it : resourceGroups) {
- Json::Value jsonResourceGroup = Json::objectValue;
- Json::Value requirements = Json::arrayValue;
- for (auto const& it2 : it) {
- Json::Value res = Json::objectValue;
- res[".type"] = it2.ResourceType;
- // res[".units"] = it2.UnitsNeeded; // Intentionally commented out
- res["slots"] = it2.SlotsNeeded;
- requirements.append(res);
- }
- jsonResourceGroup["requirements"] = requirements;
- jsonResourceGroups.append(jsonResourceGroup);
- }
- return jsonResourceGroups;
- }
- static Json::Value DumpCTestProperty(std::string const& name,
- Json::Value value)
- {
- Json::Value property = Json::objectValue;
- property["name"] = name;
- property["value"] = std::move(value);
- return property;
- }
- static Json::Value DumpCTestProperties(
- cmCTestTestHandler::cmCTestTestProperties& testProperties)
- {
- Json::Value properties = Json::arrayValue;
- if (!testProperties.AttachOnFail.empty()) {
- properties.append(DumpCTestProperty(
- "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
- }
- if (!testProperties.AttachedFiles.empty()) {
- properties.append(DumpCTestProperty(
- "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
- }
- if (testProperties.Cost != 0.0f) {
- properties.append(
- DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
- }
- if (!testProperties.Depends.empty()) {
- properties.append(
- DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
- }
- if (testProperties.Disabled) {
- properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
- }
- if (!testProperties.Environment.empty()) {
- properties.append(DumpCTestProperty(
- "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
- }
- if (!testProperties.EnvironmentModification.empty()) {
- properties.append(DumpCTestProperty(
- "ENVIRONMENT_MODIFICATION",
- DumpToJsonArray(testProperties.EnvironmentModification)));
- }
- if (!testProperties.ErrorRegularExpressions.empty()) {
- properties.append(DumpCTestProperty(
- "FAIL_REGULAR_EXPRESSION",
- DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
- }
- if (!testProperties.SkipRegularExpressions.empty()) {
- properties.append(DumpCTestProperty(
- "SKIP_REGULAR_EXPRESSION",
- DumpRegExToJsonArray(testProperties.SkipRegularExpressions)));
- }
- if (!testProperties.FixturesCleanup.empty()) {
- properties.append(DumpCTestProperty(
- "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
- }
- if (!testProperties.FixturesRequired.empty()) {
- properties.append(DumpCTestProperty(
- "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
- }
- if (!testProperties.FixturesSetup.empty()) {
- properties.append(DumpCTestProperty(
- "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
- }
- if (!testProperties.Labels.empty()) {
- properties.append(
- DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
- }
- if (!testProperties.Measurements.empty()) {
- properties.append(DumpCTestProperty(
- "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
- }
- if (!testProperties.RequiredRegularExpressions.empty()) {
- properties.append(DumpCTestProperty(
- "PASS_REGULAR_EXPRESSION",
- DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
- }
- if (!testProperties.ResourceGroups.empty()) {
- properties.append(DumpCTestProperty(
- "RESOURCE_GROUPS",
- DumpResourceGroupsToJsonArray(testProperties.ResourceGroups)));
- }
- if (testProperties.WantAffinity) {
- properties.append(
- DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
- }
- if (testProperties.Processors != 1) {
- properties.append(
- DumpCTestProperty("PROCESSORS", testProperties.Processors));
- }
- if (!testProperties.RequiredFiles.empty()) {
- properties.append(DumpCTestProperty(
- "REQUIRED_FILES", DumpToJsonArray(testProperties.RequiredFiles)));
- }
- if (!testProperties.ProjectResources.empty()) {
- properties.append(DumpCTestProperty(
- "RESOURCE_LOCK", DumpToJsonArray(testProperties.ProjectResources)));
- }
- if (testProperties.RunSerial) {
- properties.append(
- DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
- }
- if (testProperties.SkipReturnCode != -1) {
- properties.append(
- DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
- }
- if (testProperties.Timeout) {
- properties.append(
- DumpCTestProperty("TIMEOUT", testProperties.Timeout->count()));
- }
- if (!testProperties.TimeoutRegularExpressions.empty()) {
- properties.append(DumpCTestProperty(
- "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
- }
- if (testProperties.WillFail) {
- properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
- }
- if (!testProperties.Directory.empty()) {
- properties.append(
- DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
- }
- return properties;
- }
- class BacktraceData
- {
- std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
- std::unordered_map<std::string, Json::ArrayIndex> FileMap;
- std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
- Json::Value Commands = Json::arrayValue;
- Json::Value Files = Json::arrayValue;
- Json::Value Nodes = Json::arrayValue;
- Json::ArrayIndex AddCommand(std::string const& command)
- {
- auto i = this->CommandMap.find(command);
- if (i == this->CommandMap.end()) {
- i = this->CommandMap.emplace(command, this->Commands.size()).first;
- this->Commands.append(command);
- }
- return i->second;
- }
- Json::ArrayIndex AddFile(std::string const& file)
- {
- auto i = this->FileMap.find(file);
- if (i == this->FileMap.end()) {
- i = this->FileMap.emplace(file, this->Files.size()).first;
- this->Files.append(file);
- }
- return i->second;
- }
- public:
- bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
- Json::Value Dump();
- };
- bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
- {
- if (bt.Empty()) {
- return false;
- }
- cmListFileContext const* top = &bt.Top();
- auto found = this->NodeMap.find(top);
- if (found != this->NodeMap.end()) {
- index = found->second;
- return true;
- }
- Json::Value entry = Json::objectValue;
- entry["file"] = this->AddFile(top->FilePath);
- if (top->Line) {
- entry["line"] = static_cast<int>(top->Line);
- }
- if (!top->Name.empty()) {
- entry["command"] = this->AddCommand(top->Name);
- }
- Json::ArrayIndex parent;
- if (this->Add(bt.Pop(), parent)) {
- entry["parent"] = parent;
- }
- index = this->NodeMap[top] = this->Nodes.size();
- this->Nodes.append(std::move(entry)); // NOLINT(*)
- return true;
- }
- Json::Value BacktraceData::Dump()
- {
- Json::Value backtraceGraph;
- this->CommandMap.clear();
- this->FileMap.clear();
- this->NodeMap.clear();
- backtraceGraph["commands"] = std::move(this->Commands);
- backtraceGraph["files"] = std::move(this->Files);
- backtraceGraph["nodes"] = std::move(this->Nodes);
- return backtraceGraph;
- }
- static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
- cmListFileBacktrace const& bt)
- {
- Json::ArrayIndex backtrace;
- if (backtraceGraph.Add(bt, backtrace)) {
- object["backtrace"] = backtrace;
- }
- }
- static Json::Value DumpCTestInfo(
- cmCTestRunTest& testRun,
- cmCTestTestHandler::cmCTestTestProperties& testProperties,
- BacktraceData& backtraceGraph)
- {
- Json::Value testInfo = Json::objectValue;
- // test name should always be present
- testInfo["name"] = testProperties.Name;
- std::string const& config = testRun.GetCTest()->GetConfigType();
- if (!config.empty()) {
- testInfo["config"] = config;
- }
- std::string const& command = testRun.GetActualCommand();
- if (!command.empty()) {
- std::vector<std::string> commandAndArgs;
- commandAndArgs.push_back(command);
- const std::vector<std::string>& args = testRun.GetArguments();
- if (!args.empty()) {
- commandAndArgs.reserve(args.size() + 1);
- cm::append(commandAndArgs, args);
- }
- testInfo["command"] = DumpToJsonArray(commandAndArgs);
- }
- Json::Value properties = DumpCTestProperties(testProperties);
- if (!properties.empty()) {
- testInfo["properties"] = properties;
- }
- if (!testProperties.Backtrace.Empty()) {
- AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
- }
- return testInfo;
- }
- static Json::Value DumpVersion(int major, int minor)
- {
- Json::Value version = Json::objectValue;
- version["major"] = major;
- version["minor"] = minor;
- return version;
- }
- void cmCTestMultiProcessHandler::PrintOutputAsJson()
- {
- this->TestHandler->SetMaxIndex(this->FindMaxIndex());
- Json::Value result = Json::objectValue;
- result["kind"] = "ctestInfo";
- result["version"] = DumpVersion(1, 0);
- BacktraceData backtraceGraph;
- Json::Value tests = Json::arrayValue;
- for (auto& it : this->Properties) {
- cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
- // Don't worry if this fails, we are only showing the test list, not
- // running the tests
- cmWorkingDirectory workdir(p.Directory);
- cmCTestRunTest testRun(*this, p.Index);
- testRun.ComputeArguments();
- // Skip tests not available in this configuration.
- if (p.Args.size() >= 2 && p.Args[1] == "NOT_AVAILABLE") {
- continue;
- }
- Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
- tests.append(testInfo);
- }
- result["backtraceGraph"] = backtraceGraph.Dump();
- result["tests"] = std::move(tests);
- Json::StreamWriterBuilder builder;
- builder["indentation"] = " ";
- std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
- jout->write(result, &std::cout);
- }
- // For ShowOnly mode
- void cmCTestMultiProcessHandler::PrintTestList()
- {
- if (this->CTest->GetOutputAsJson()) {
- this->PrintOutputAsJson();
- return;
- }
- this->TestHandler->SetMaxIndex(this->FindMaxIndex());
- for (auto& it : this->Properties) {
- cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
- // Don't worry if this fails, we are only showing the test list, not
- // running the tests
- cmWorkingDirectory workdir(p.Directory);
- cmCTestRunTest testRun(*this, p.Index);
- testRun.ComputeArguments(); // logs the command in verbose mode
- if (!p.Labels.empty()) // print the labels
- {
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "Labels:", this->Quiet);
- }
- for (std::string const& label : p.Labels) {
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, " " << label,
- this->Quiet);
- }
- if (!p.Labels.empty()) // print the labels
- {
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl,
- this->Quiet);
- }
- if (this->TestHandler->MemCheck) {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Memory Check",
- this->Quiet);
- } else {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Test", this->Quiet);
- }
- std::ostringstream indexStr;
- indexStr << " #" << p.Index << ":";
- cmCTestOptionalLog(
- this->CTest, HANDLER_OUTPUT,
- std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex()))
- << indexStr.str(),
- this->Quiet);
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << p.Name,
- this->Quiet);
- if (p.Disabled) {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " (Disabled)",
- this->Quiet);
- }
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, std::endl, this->Quiet);
- }
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
- std::endl
- << "Total Tests: " << this->Total << std::endl,
- this->Quiet);
- }
- void cmCTestMultiProcessHandler::PrintLabels()
- {
- std::set<std::string> allLabels;
- for (auto& it : this->Properties) {
- cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
- allLabels.insert(p.Labels.begin(), p.Labels.end());
- }
- if (!allLabels.empty()) {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "All Labels:" << std::endl,
- this->Quiet);
- } else {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
- "No Labels Exist" << std::endl, this->Quiet);
- }
- for (std::string const& label : allLabels) {
- cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << label << std::endl,
- this->Quiet);
- }
- }
- void cmCTestMultiProcessHandler::CheckResume()
- {
- std::string fname =
- this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
- if (this->CTest->GetFailover()) {
- if (cmSystemTools::FileExists(fname, true)) {
- *this->TestHandler->LogFile
- << "Resuming previously interrupted test set" << std::endl
- << "----------------------------------------------------------"
- << std::endl;
- cmsys::ifstream fin;
- fin.open(fname.c_str());
- std::string line;
- while (std::getline(fin, line)) {
- int index = atoi(line.c_str());
- this->RemoveTest(index);
- }
- fin.close();
- }
- } else if (cmSystemTools::FileExists(fname, true)) {
- cmSystemTools::RemoveFile(fname);
- }
- }
- void cmCTestMultiProcessHandler::RemoveTest(int index)
- {
- this->OrderedTests.erase(
- std::find(this->OrderedTests.begin(), this->OrderedTests.end(), index));
- this->PendingTests.erase(index);
- this->Properties.erase(index);
- this->Completed++;
- }
- int cmCTestMultiProcessHandler::FindMaxIndex()
- {
- int max = 0;
- for (auto const& i : this->PendingTests) {
- if (i.first > max) {
- max = i.first;
- }
- }
- return max;
- }
- // Returns true if no cycles exist in the dependency graph
- bool cmCTestMultiProcessHandler::CheckCycles()
- {
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "Checking test dependency graph..." << std::endl,
- this->Quiet);
- for (auto const& it : this->PendingTests) {
- // DFS from each element to itself
- int root = it.first;
- std::set<int> visited;
- std::stack<int> s;
- s.push(root);
- while (!s.empty()) {
- int test = s.top();
- s.pop();
- if (visited.insert(test).second) {
- for (auto const& d : this->PendingTests[test].Depends) {
- if (d == root) {
- // cycle exists
- cmCTestLog(
- this->CTest, ERROR_MESSAGE,
- "Error: a cycle exists in the test dependency graph "
- "for the test \""
- << this->Properties[root]->Name
- << "\".\nPlease fix the cycle and run ctest again.\n");
- return false;
- }
- s.push(d);
- }
- }
- }
- }
- cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
- "Checking test dependency graph end" << std::endl,
- this->Quiet);
- return true;
- }
- bool cmCTestMultiProcessHandler::CheckGeneratedResourceSpec()
- {
- for (auto& test : this->Properties) {
- if (!test.second->GeneratedResourceSpecFile.empty()) {
- if (this->ResourceSpecSetupTest) {
- cmCTestLog(
- this->CTest, ERROR_MESSAGE,
- "Only one test may define the GENERATED_RESOURCE_SPEC_FILE property"
- << std::endl);
- return false;
- }
- if (test.second->FixturesSetup.size() != 1) {
- cmCTestLog(this->CTest, ERROR_MESSAGE,
- "Test that defines GENERATED_RESOURCE_SPEC_FILE must have "
- "exactly one FIXTURES_SETUP"
- << std::endl);
- return false;
- }
- if (!cmSystemTools::FileIsFullPath(
- test.second->GeneratedResourceSpecFile)) {
- cmCTestLog(this->CTest, ERROR_MESSAGE,
- "GENERATED_RESOURCE_SPEC_FILE must be an absolute path"
- << std::endl);
- return false;
- }
- this->ResourceSpecSetupTest = test.first;
- this->ResourceSpecSetupFixture = *test.second->FixturesSetup.begin();
- }
- }
- if (!this->ResourceSpecSetupFixture.empty()) {
- for (auto& test : this->Properties) {
- if (!test.second->ResourceGroups.empty() &&
- !test.second->FixturesRequired.count(
- this->ResourceSpecSetupFixture)) {
- cmCTestLog(this->CTest, ERROR_MESSAGE,
- "All tests that have RESOURCE_GROUPS must include the "
- "resource spec generator fixture in their FIXTURES_REQUIRED"
- << std::endl);
- return false;
- }
- }
- }
- if (!this->ResourceSpecFile.empty()) {
- if (this->ResourceSpecSetupTest) {
- cmCTestLog(this->CTest, ERROR_MESSAGE,
- "GENERATED_RESOURCE_SPEC_FILE test property cannot be used "
- "in conjunction with ResourceSpecFile option"
- << std::endl);
- return false;
- }
- std::string error;
- if (!this->InitResourceAllocator(error)) {
- cmCTestLog(this->CTest, ERROR_MESSAGE, error << std::endl);
- return false;
- }
- }
- return true;
- }
- bool cmCTestMultiProcessHandler::InitResourceAllocator(std::string& error)
- {
- if (!this->ResourceSpec.ReadFromJSONFile(this->ResourceSpecFile)) {
- error = cmStrCat("Could not read/parse resource spec file ",
- this->ResourceSpecFile, ": ",
- this->ResourceSpec.parseState.GetErrorMessage());
- return false;
- }
- this->UseResourceSpec = true;
- this->ResourceAllocator.InitializeFromResourceSpec(this->ResourceSpec);
- return true;
- }
|