cmCTestTestHandler.cxx 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657
  1. /*=========================================================================
  2. Program: CMake - Cross-Platform Makefile Generator
  3. Module: $RCSfile$
  4. Language: C++
  5. Date: $Date$
  6. Version: $Revision$
  7. Copyright (c) 2002 Kitware, Inc., Insight Consortium. All rights reserved.
  8. See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
  9. This software is distributed WITHOUT ANY WARRANTY; without even
  10. the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  11. PURPOSE. See the above copyright notices for more information.
  12. =========================================================================*/
  13. #include "cmCTestTestHandler.h"
  14. #include "cmCTest.h"
  15. #include "cmake.h"
  16. #include <cmsys/Process.h>
  17. #include <cmsys/RegularExpression.hxx>
  18. #include <cmsys/Base64.h>
  19. #include "cmMakefile.h"
  20. #include <stdlib.h>
  21. #include <math.h>
  22. #include <float.h>
  23. static const char* cmCTestMemCheckResultStrings[] = {
  24. "ABR",
  25. "ABW",
  26. "ABWL",
  27. "COR",
  28. "EXU",
  29. "FFM",
  30. "FIM",
  31. "FMM",
  32. "FMR",
  33. "FMW",
  34. "FUM",
  35. "IPR",
  36. "IPW",
  37. "MAF",
  38. "MLK",
  39. "MPK",
  40. "NPR",
  41. "ODS",
  42. "PAR",
  43. "PLK",
  44. "UMC",
  45. "UMR",
  46. 0
  47. };
  48. static const char* cmCTestMemCheckResultLongStrings[] = {
  49. "Threading Problem",
  50. "ABW",
  51. "ABWL",
  52. "COR",
  53. "EXU",
  54. "FFM",
  55. "FIM",
  56. "Mismatched deallocation",
  57. "FMR",
  58. "FMW",
  59. "FUM",
  60. "IPR",
  61. "IPW",
  62. "MAF",
  63. "Memory Leak",
  64. "Potential Memory Leak",
  65. "NPR",
  66. "ODS",
  67. "Invalid syscall param",
  68. "PLK",
  69. "Uninitialized Memory Conditional",
  70. "Uninitialized Memory Read",
  71. 0
  72. };
  73. bool TryExecutable(const char *dir, const char *file,
  74. std::string *fullPath, const char *subdir)
  75. {
  76. // try current directory
  77. std::string tryPath;
  78. if (dir && strcmp(dir,""))
  79. {
  80. tryPath = dir;
  81. tryPath += "/";
  82. }
  83. if (subdir && strcmp(subdir,""))
  84. {
  85. tryPath += subdir;
  86. tryPath += "/";
  87. }
  88. tryPath += file;
  89. if(cmSystemTools::FileExists(tryPath.c_str()))
  90. {
  91. *fullPath = cmSystemTools::CollapseFullPath(tryPath.c_str());
  92. return true;
  93. }
  94. tryPath += cmSystemTools::GetExecutableExtension();
  95. if(cmSystemTools::FileExists(tryPath.c_str()))
  96. {
  97. *fullPath = cmSystemTools::CollapseFullPath(tryPath.c_str());
  98. return true;
  99. }
  100. return false;
  101. }
  102. // get the next number in a string with numbers separated by ,
  103. // pos is the start of the search and pos2 is the end of the search
  104. // pos becomes pos2 after a call to GetNextNumber.
  105. // -1 is returned at the end of the list.
  106. inline int GetNextNumber(std::string const& in,
  107. int& val,
  108. std::string::size_type& pos,
  109. std::string::size_type& pos2)
  110. {
  111. pos2 = in.find(',', pos);
  112. if(pos2 != in.npos)
  113. {
  114. if(pos2-pos == 0)
  115. {
  116. val = -1;
  117. }
  118. else
  119. {
  120. val = atoi(in.substr(pos, pos2-pos).c_str());
  121. }
  122. pos = pos2+1;
  123. return 1;
  124. }
  125. else
  126. {
  127. if(in.size()-pos == 0)
  128. {
  129. val = -1;
  130. }
  131. else
  132. {
  133. val = atoi(in.substr(pos, in.size()-pos).c_str());
  134. }
  135. return 0;
  136. }
  137. }
  138. // get the next number in a string with numbers separated by ,
  139. // pos is the start of the search and pos2 is the end of the search
  140. // pos becomes pos2 after a call to GetNextNumber.
  141. // -1 is returned at the end of the list.
  142. inline int GetNextRealNumber(std::string const& in,
  143. double& val,
  144. std::string::size_type& pos,
  145. std::string::size_type& pos2)
  146. {
  147. pos2 = in.find(',', pos);
  148. if(pos2 != in.npos)
  149. {
  150. if(pos2-pos == 0)
  151. {
  152. val = -1;
  153. }
  154. else
  155. {
  156. val = atof(in.substr(pos, pos2-pos).c_str());
  157. }
  158. pos = pos2+1;
  159. return 1;
  160. }
  161. else
  162. {
  163. if(in.size()-pos == 0)
  164. {
  165. val = -1;
  166. }
  167. else
  168. {
  169. val = atof(in.substr(pos, in.size()-pos).c_str());
  170. }
  171. return 0;
  172. }
  173. }
  174. //----------------------------------------------------------------------
  175. cmCTestTestHandler::cmCTestTestHandler()
  176. {
  177. m_Verbose = false;
  178. m_CTest = 0;
  179. m_UseIncludeRegExp = false;
  180. m_UseExcludeRegExp = false;
  181. m_UseExcludeRegExpFirst = false;
  182. m_MaximumPassedTestResultSize = 100 * 1024;
  183. m_MaximumFailedTestResultSize = 200 * 1024;
  184. }
  185. //----------------------------------------------------------------------
  186. void cmCTestTestHandler::PopulateCustomVectors(cmMakefile *mf)
  187. {
  188. cmCTest::PopulateCustomVector(mf, "CTEST_CUSTOM_PRE_TEST",
  189. m_CustomPreTest);
  190. cmCTest::PopulateCustomVector(mf, "CTEST_CUSTOM_POST_TEST",
  191. m_CustomPostTest);
  192. cmCTest::PopulateCustomVector(mf, "CTEST_CUSTOM_PRE_MEMCHECK",
  193. m_CustomPreMemCheck);
  194. cmCTest::PopulateCustomVector(mf, "CTEST_CUSTOM_POST_MEMCHECK",
  195. m_CustomPostMemCheck);
  196. const char* maxstr =
  197. mf->GetDefinition("CTEST_CUSTOM_PASSED_TEST_STRING_MAXLEN");
  198. if ( maxstr )
  199. {
  200. long val = atoi(maxstr);
  201. if ( val > 0 )
  202. {
  203. m_MaximumPassedTestResultSize = val;
  204. }
  205. }
  206. maxstr = mf->GetDefinition("CTEST_CUSTOM_FAILED_TEST_STRING_MAXLEN");
  207. if ( maxstr )
  208. {
  209. long val = atoi(maxstr);
  210. if ( val > 0 )
  211. {
  212. m_MaximumFailedTestResultSize = val;
  213. }
  214. }
  215. cmCTest::PopulateCustomVector(mf,
  216. "CTEST_CUSTOM_TESTS_IGNORE",
  217. m_CustomTestsIgnore);
  218. cmCTest::PopulateCustomVector(mf,
  219. "CTEST_CUSTOM_MEMCHECK_IGNORE",
  220. m_CustomMemCheckIgnore);
  221. }
  222. //----------------------------------------------------------------------
  223. //clearly it would be nice if this were broken up into a few smaller
  224. //functions and commented...
  225. int cmCTestTestHandler::TestDirectory(cmCTest *ctest_inst, bool memcheck)
  226. {
  227. m_CTest = ctest_inst;
  228. m_TestResults.clear();
  229. std::cout << (memcheck ? "Memory check" : "Test") << " project" << std::endl;
  230. if ( memcheck )
  231. {
  232. if ( !this->InitializeMemoryChecking() )
  233. {
  234. return 1;
  235. }
  236. }
  237. if ( memcheck )
  238. {
  239. if ( !this->ExecuteCommands(m_CustomPreMemCheck) )
  240. {
  241. std::cerr << "Problem executing pre-memcheck command(s)." << std::endl;
  242. return 1;
  243. }
  244. }
  245. else
  246. {
  247. if ( !this->ExecuteCommands(m_CustomPreTest) )
  248. {
  249. std::cerr << "Problem executing pre-test command(s)." << std::endl;
  250. return 1;
  251. }
  252. }
  253. std::vector<cmStdString> passed;
  254. std::vector<cmStdString> failed;
  255. int total;
  256. this->ProcessDirectory(passed, failed, memcheck);
  257. total = int(passed.size()) + int(failed.size());
  258. if (total == 0)
  259. {
  260. if ( !m_CTest->GetShowOnly() )
  261. {
  262. std::cerr << "No tests were found!!!\n";
  263. }
  264. }
  265. else
  266. {
  267. if (m_Verbose && passed.size() &&
  268. (m_UseIncludeRegExp || m_UseExcludeRegExp))
  269. {
  270. std::cerr << "\nThe following tests passed:\n";
  271. for(std::vector<cmStdString>::iterator j = passed.begin();
  272. j != passed.end(); ++j)
  273. {
  274. std::cerr << "\t" << *j << "\n";
  275. }
  276. }
  277. float percent = float(passed.size()) * 100.0f / total;
  278. if ( failed.size() > 0 && percent > 99)
  279. {
  280. percent = 99;
  281. }
  282. fprintf(stderr,"\n%.0f%% tests passed, %i tests failed out of %i\n",
  283. percent, int(failed.size()), total);
  284. if (failed.size())
  285. {
  286. std::ofstream ofs;
  287. std::cerr << "\nThe following tests FAILED:\n";
  288. m_CTest->OpenOutputFile("Temporary", "LastTestsFailed.log", ofs);
  289. std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
  290. for(ftit = m_TestResults.begin();
  291. ftit != m_TestResults.end(); ++ftit)
  292. {
  293. if ( ftit->m_Status != cmCTestTestHandler::COMPLETED )
  294. {
  295. ofs << ftit->m_TestCount << ":" << ftit->m_Name << std::endl;
  296. fprintf(stderr, "\t%3d - %s (%s)\n", ftit->m_TestCount, ftit->m_Name.c_str(),
  297. this->GetTestStatus(ftit->m_Status));
  298. }
  299. }
  300. }
  301. }
  302. if ( m_CTest->GetProduceXML() )
  303. {
  304. std::ofstream xmlfile;
  305. if( !m_CTest->OpenOutputFile(m_CTest->GetCurrentTag(),
  306. (memcheck ? "DynamicAnalysis.xml" : "Test.xml"), xmlfile) )
  307. {
  308. std::cerr << "Cannot create " << (memcheck ? "memory check" : "testing")
  309. << " XML file" << std::endl;
  310. return 1;
  311. }
  312. if ( memcheck )
  313. {
  314. this->GenerateDartMemCheckOutput(xmlfile);
  315. }
  316. else
  317. {
  318. this->GenerateDartTestOutput(xmlfile);
  319. }
  320. }
  321. if ( memcheck )
  322. {
  323. if ( !this->ExecuteCommands(m_CustomPostMemCheck) )
  324. {
  325. std::cerr << "Problem executing post-memcheck command(s)." << std::endl;
  326. return 1;
  327. }
  328. }
  329. else
  330. {
  331. if ( !this->ExecuteCommands(m_CustomPostTest) )
  332. {
  333. std::cerr << "Problem executing post-test command(s)." << std::endl;
  334. return 1;
  335. }
  336. }
  337. return int(failed.size());
  338. }
  339. void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
  340. std::vector<cmStdString> &failed,
  341. bool memcheck)
  342. {
  343. std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
  344. cmsys::RegularExpression dartStuff("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)");
  345. tm_ListOfTests testlist;
  346. this->GetListOfTests(&testlist, memcheck);
  347. tm_ListOfTests::size_type tmsize = testlist.size();
  348. std::ofstream ofs;
  349. std::ofstream *olog = 0;
  350. if ( !m_CTest->GetShowOnly() && tmsize > 0 &&
  351. m_CTest->OpenOutputFile("Temporary",
  352. (memcheck?"LastMemCheck.log":"LastTest.log"), ofs) )
  353. {
  354. olog = &ofs;
  355. }
  356. m_StartTest = m_CTest->CurrentTime();
  357. double elapsed_time_start = cmSystemTools::GetTime();
  358. if ( olog )
  359. {
  360. *olog << "Start testing: " << m_StartTest << std::endl
  361. << "----------------------------------------------------------"
  362. << std::endl;
  363. }
  364. // expand the test list
  365. this->ExpandTestsToRunInformation((int)tmsize);
  366. int cnt = 0;
  367. tm_ListOfTests::iterator it;
  368. std::string last_directory = "";
  369. for ( it = testlist.begin(); it != testlist.end(); it ++ )
  370. {
  371. cnt ++;
  372. const std::string& testname = it->m_Name;
  373. tm_VectorOfListFileArgs& args = it->m_Args;
  374. cmCTestTestResult cres;
  375. cres.m_Status = cmCTestTestHandler::NOT_RUN;
  376. cres.m_TestCount = cnt;
  377. if (!(last_directory == it->m_Directory))
  378. {
  379. if ( m_Verbose )
  380. {
  381. std::cerr << "Changing directory into "
  382. << it->m_Directory.c_str() << "\n";
  383. }
  384. last_directory = it->m_Directory;
  385. cmSystemTools::ChangeDirectory(it->m_Directory.c_str());
  386. }
  387. cres.m_Name = testname;
  388. if(m_TestsToRun.size() &&
  389. std::find(m_TestsToRun.begin(), m_TestsToRun.end(), cnt) == m_TestsToRun.end())
  390. {
  391. continue;
  392. }
  393. if ( m_CTest->GetShowOnly() )
  394. {
  395. std::cerr.width(3);
  396. std::cerr << cnt << "/";
  397. std::cerr.width(3);
  398. std::cerr << tmsize << " Testing ";
  399. std::string outname = testname;
  400. outname.resize(30, ' ');
  401. std::cerr << outname.c_str() << "\n";
  402. }
  403. else
  404. {
  405. std::cerr.width(3);
  406. std::cerr << cnt << "/";
  407. std::cerr.width(3);
  408. std::cerr << tmsize << " Testing ";
  409. std::string outname = testname;
  410. outname.resize(30, ' ');
  411. std::cerr << outname.c_str();
  412. std::cerr.flush();
  413. }
  414. //std::cerr << "Testing " << args[0] << " ... ";
  415. // find the test executable
  416. std::string actualCommand = this->FindTheExecutable(args[1].Value.c_str());
  417. std::string testCommand = cmSystemTools::ConvertToOutputPath(actualCommand.c_str());
  418. std::string memcheckcommand = "";
  419. // continue if we did not find the executable
  420. if (testCommand == "")
  421. {
  422. std::cerr << "Unable to find executable: " <<
  423. args[1].Value.c_str() << "\n";
  424. if ( !m_CTest->GetShowOnly() )
  425. {
  426. m_TestResults.push_back( cres );
  427. failed.push_back(testname);
  428. continue;
  429. }
  430. }
  431. // add the arguments
  432. tm_VectorOfListFileArgs::const_iterator j = args.begin();
  433. ++j;
  434. ++j;
  435. std::vector<const char*> arguments;
  436. if ( memcheck )
  437. {
  438. std::vector<cmStdString>::size_type pp;
  439. arguments.push_back(m_MemoryTester.c_str());
  440. memcheckcommand = m_MemoryTester;
  441. for ( pp = 0; pp < m_MemoryTesterOptionsParsed.size(); pp ++ )
  442. {
  443. arguments.push_back(m_MemoryTesterOptionsParsed[pp].c_str());
  444. memcheckcommand += " ";
  445. memcheckcommand += cmSystemTools::EscapeSpaces(m_MemoryTesterOptionsParsed[pp].c_str());
  446. }
  447. }
  448. arguments.push_back(actualCommand.c_str());
  449. for(;j != args.end(); ++j)
  450. {
  451. testCommand += " ";
  452. testCommand += cmSystemTools::EscapeSpaces(j->Value.c_str());
  453. arguments.push_back(j->Value.c_str());
  454. }
  455. arguments.push_back(0);
  456. /**
  457. * Run an executable command and put the stdout in output.
  458. */
  459. std::string output;
  460. int retVal = 0;
  461. if ( m_Verbose )
  462. {
  463. std::cout << std::endl << (memcheck?"MemCheck":"Test") << " command: " << testCommand << std::endl;
  464. if ( memcheck )
  465. {
  466. std::cout << "Memory check command: " << memcheckcommand << std::endl;
  467. }
  468. }
  469. if ( olog )
  470. {
  471. *olog << cnt << "/" << tmsize
  472. << " Test: " << testname.c_str() << std::endl;
  473. *olog << "Command: ";
  474. std::vector<cmStdString>::size_type ll;
  475. for ( ll = 0; ll < arguments.size()-1; ll ++ )
  476. {
  477. *olog << "\"" << arguments[ll] << "\" ";
  478. }
  479. *olog
  480. << std::endl
  481. << "Directory: " << it->m_Directory << std::endl
  482. << "\"" << testname.c_str() << "\" start time: "
  483. << m_CTest->CurrentTime() << std::endl
  484. << "Output:" << std::endl
  485. << "----------------------------------------------------------"
  486. << std::endl;
  487. }
  488. int res = 0;
  489. double clock_start, clock_finish;
  490. clock_start = cmSystemTools::GetTime();
  491. if ( !m_CTest->GetShowOnly() )
  492. {
  493. res = m_CTest->RunTest(arguments, &output, &retVal, olog);
  494. }
  495. clock_finish = cmSystemTools::GetTime();
  496. if ( olog )
  497. {
  498. double ttime = clock_finish - clock_start;
  499. int hours = static_cast<int>(ttime / (60 * 60));
  500. int minutes = static_cast<int>(ttime / 60) % 60;
  501. int seconds = static_cast<int>(ttime) % 60;
  502. char buffer[100];
  503. sprintf(buffer, "%02d:%02d:%02d", hours, minutes, seconds);
  504. *olog
  505. << "----------------------------------------------------------"
  506. << std::endl
  507. << "\"" << testname.c_str() << "\" end time: "
  508. << m_CTest->CurrentTime() << std::endl
  509. << "\"" << testname.c_str() << "\" time elapsed: "
  510. << buffer << std::endl
  511. << "----------------------------------------------------------"
  512. << std::endl << std::endl;
  513. }
  514. cres.m_ExecutionTime = (double)(clock_finish - clock_start);
  515. cres.m_FullCommandLine = testCommand;
  516. if ( !m_CTest->GetShowOnly() )
  517. {
  518. if (res == cmsysProcess_State_Exited && retVal == 0)
  519. {
  520. std::cerr << " Passed\n";
  521. passed.push_back(testname);
  522. cres.m_Status = cmCTestTestHandler::COMPLETED;
  523. }
  524. else
  525. {
  526. cres.m_Status = cmCTestTestHandler::FAILED;
  527. if ( res == cmsysProcess_State_Expired )
  528. {
  529. std::cerr << "***Timeout\n";
  530. cres.m_Status = cmCTestTestHandler::TIMEOUT;
  531. }
  532. else if ( res == cmsysProcess_State_Exception )
  533. {
  534. std::cerr << "***Exception: ";
  535. switch ( retVal )
  536. {
  537. case cmsysProcess_Exception_Fault:
  538. std::cerr << "SegFault";
  539. cres.m_Status = cmCTestTestHandler::SEGFAULT;
  540. break;
  541. case cmsysProcess_Exception_Illegal:
  542. std::cerr << "Illegal";
  543. cres.m_Status = cmCTestTestHandler::ILLEGAL;
  544. break;
  545. case cmsysProcess_Exception_Interrupt:
  546. std::cerr << "Interrupt";
  547. cres.m_Status = cmCTestTestHandler::INTERRUPT;
  548. break;
  549. case cmsysProcess_Exception_Numerical:
  550. std::cerr << "Numerical";
  551. cres.m_Status = cmCTestTestHandler::NUMERICAL;
  552. break;
  553. default:
  554. std::cerr << "Other";
  555. cres.m_Status = cmCTestTestHandler::OTHER_FAULT;
  556. }
  557. std::cerr << "\n";
  558. }
  559. else if ( res == cmsysProcess_State_Error )
  560. {
  561. std::cerr << "***Bad command " << res << "\n";
  562. cres.m_Status = cmCTestTestHandler::BAD_COMMAND;
  563. }
  564. else
  565. {
  566. std::cerr << "***Failed\n";
  567. }
  568. failed.push_back(testname);
  569. }
  570. if (output != "")
  571. {
  572. if (dartStuff.find(output.c_str()))
  573. {
  574. std::string dartString = dartStuff.match(1);
  575. cmSystemTools::ReplaceString(output, dartString.c_str(),"");
  576. cres.m_RegressionImages = this->GenerateRegressionImages(dartString);
  577. }
  578. }
  579. }
  580. cres.m_Output = output;
  581. cres.m_ReturnValue = retVal;
  582. std::string nwd = it->m_Directory;
  583. if ( nwd.size() > m_CTest->GetToplevelPath().size() )
  584. {
  585. nwd = "." + nwd.substr(m_CTest->GetToplevelPath().size(), nwd.npos);
  586. }
  587. cmSystemTools::ReplaceString(nwd, "\\", "/");
  588. cres.m_Path = nwd;
  589. cres.m_CompletionStatus = "Completed";
  590. m_TestResults.push_back( cres );
  591. }
  592. m_EndTest = m_CTest->CurrentTime();
  593. m_ElapsedTestingTime = cmSystemTools::GetTime() - elapsed_time_start;
  594. if ( olog )
  595. {
  596. *olog << "End testing: " << m_EndTest << std::endl;
  597. }
  598. cmSystemTools::ChangeDirectory(current_dir.c_str());
  599. }
  600. void cmCTestTestHandler::GenerateDartMemCheckOutput(std::ostream& os)
  601. {
  602. if ( !m_CTest->GetProduceXML() )
  603. {
  604. return;
  605. }
  606. m_CTest->StartXML(os);
  607. os << "<DynamicAnalysis Checker=\"";
  608. switch ( m_MemoryTesterStyle )
  609. {
  610. case cmCTestTestHandler::VALGRIND:
  611. os << "Valgrind";
  612. break;
  613. case cmCTestTestHandler::PURIFY:
  614. os << "Purify";
  615. break;
  616. case cmCTestTestHandler::BOUNDS_CHECKER:
  617. os << "BoundsChecker";
  618. break;
  619. default:
  620. os << "Unknown";
  621. }
  622. os << "\">" << std::endl;
  623. os << "\t<StartDateTime>" << m_StartTest << "</StartDateTime>\n"
  624. << "\t<TestList>\n";
  625. tm_TestResultsVector::size_type cc;
  626. for ( cc = 0; cc < m_TestResults.size(); cc ++ )
  627. {
  628. cmCTestTestResult *result = &m_TestResults[cc];
  629. os << "\t\t<Test>" << cmCTest::MakeXMLSafe(result->m_Path)
  630. << "/" << cmCTest::MakeXMLSafe(result->m_Name)
  631. << "</Test>" << std::endl;
  632. }
  633. os << "\t</TestList>\n";
  634. std::cout << "-- Processing memory checking output: ";
  635. unsigned int total = m_TestResults.size();
  636. unsigned int step = total / 10;
  637. unsigned int current = 0;
  638. for ( cc = 0; cc < m_TestResults.size(); cc ++ )
  639. {
  640. cmCTestTestResult *result = &m_TestResults[cc];
  641. std::string memcheckstr;
  642. int memcheckresults[cmCTestTestHandler::NO_MEMORY_FAULT];
  643. int kk;
  644. bool res = this->ProcessMemCheckOutput(result->m_Output, memcheckstr, memcheckresults);
  645. if ( res && result->m_Status == cmCTestTestHandler::COMPLETED )
  646. {
  647. continue;
  648. }
  649. os << "\t<Test Status=\"";
  650. if ( result->m_Status == cmCTestTestHandler::COMPLETED )
  651. {
  652. os << "passed";
  653. }
  654. else if ( result->m_Status == cmCTestTestHandler::NOT_RUN )
  655. {
  656. os << "notrun";
  657. }
  658. else
  659. {
  660. os << "failed";
  661. }
  662. os << "\">\n"
  663. << "\t\t<Name>" << cmCTest::MakeXMLSafe(result->m_Name) << "</Name>\n"
  664. << "\t\t<Path>" << cmCTest::MakeXMLSafe(result->m_Path) << "</Path>\n"
  665. << "\t\t<FullName>" << cmCTest::MakeXMLSafe(result->m_Path)
  666. << "/" << cmCTest::MakeXMLSafe(result->m_Name) << "</FullName>\n"
  667. << "\t\t<FullCommandLine>"
  668. << cmCTest::MakeXMLSafe(result->m_FullCommandLine)
  669. << "</FullCommandLine>\n"
  670. << "\t\t<Results>" << std::endl;
  671. for ( kk = 0; cmCTestMemCheckResultLongStrings[kk]; kk ++ )
  672. {
  673. if ( memcheckresults[kk] )
  674. {
  675. os << "\t\t\t<Defect type=\"" << cmCTestMemCheckResultLongStrings[kk] << "\">"
  676. << memcheckresults[kk]
  677. << "</Defect>" << std::endl;
  678. }
  679. m_MemoryTesterGlobalResults[kk] += memcheckresults[kk];
  680. }
  681. os
  682. << "\t\t</Results>\n"
  683. << "\t<Log>\n" << memcheckstr << std::endl
  684. << "\t</Log>\n"
  685. << "\t</Test>" << std::endl;
  686. if ( current < cc )
  687. {
  688. std::cout << "#";
  689. std::cout.flush();
  690. current += step;
  691. }
  692. }
  693. std::cout << std::endl;
  694. std::cerr << "Memory checking results:" << std::endl;
  695. os << "\t<DefectList>" << std::endl;
  696. for ( cc = 0; cmCTestMemCheckResultStrings[cc]; cc ++ )
  697. {
  698. if ( m_MemoryTesterGlobalResults[cc] )
  699. {
  700. std::cerr.width(35);
  701. std::cerr << cmCTestMemCheckResultLongStrings[cc] << " - "
  702. << m_MemoryTesterGlobalResults[cc] << std::endl;
  703. os << "\t\t<Defect Type=\"" << cmCTestMemCheckResultLongStrings[cc] << "\"/>" << std::endl;
  704. }
  705. }
  706. os << "\t</DefectList>" << std::endl;
  707. os << "\t<EndDateTime>" << m_EndTest << "</EndDateTime>" << std::endl;
  708. os << "<ElapsedMinutes>"
  709. << static_cast<int>(m_ElapsedTestingTime/6)/10.0
  710. << "</ElapsedMinutes>\n";
  711. os << "</DynamicAnalysis>" << std::endl;
  712. m_CTest->EndXML(os);
  713. }
  714. void cmCTestTestHandler::GenerateDartTestOutput(std::ostream& os)
  715. {
  716. if ( !m_CTest->GetProduceXML() )
  717. {
  718. return;
  719. }
  720. m_CTest->StartXML(os);
  721. os << "<Testing>\n"
  722. << "\t<StartDateTime>" << m_StartTest << "</StartDateTime>\n"
  723. << "\t<TestList>\n";
  724. tm_TestResultsVector::size_type cc;
  725. for ( cc = 0; cc < m_TestResults.size(); cc ++ )
  726. {
  727. cmCTestTestResult *result = &m_TestResults[cc];
  728. os << "\t\t<Test>" << cmCTest::MakeXMLSafe(result->m_Path)
  729. << "/" << cmCTest::MakeXMLSafe(result->m_Name)
  730. << "</Test>" << std::endl;
  731. }
  732. os << "\t</TestList>\n";
  733. for ( cc = 0; cc < m_TestResults.size(); cc ++ )
  734. {
  735. cmCTestTestResult *result = &m_TestResults[cc];
  736. os << "\t<Test Status=\"";
  737. if ( result->m_Status == cmCTestTestHandler::COMPLETED )
  738. {
  739. os << "passed";
  740. }
  741. else if ( result->m_Status == cmCTestTestHandler::NOT_RUN )
  742. {
  743. os << "notrun";
  744. }
  745. else
  746. {
  747. os << "failed";
  748. }
  749. os << "\">\n"
  750. << "\t\t<Name>" << cmCTest::MakeXMLSafe(result->m_Name) << "</Name>\n"
  751. << "\t\t<Path>" << cmCTest::MakeXMLSafe(result->m_Path) << "</Path>\n"
  752. << "\t\t<FullName>" << cmCTest::MakeXMLSafe(result->m_Path)
  753. << "/" << cmCTest::MakeXMLSafe(result->m_Name) << "</FullName>\n"
  754. << "\t\t<FullCommandLine>"
  755. << cmCTest::MakeXMLSafe(result->m_FullCommandLine)
  756. << "</FullCommandLine>\n"
  757. << "\t\t<Results>" << std::endl;
  758. if ( result->m_Status != cmCTestTestHandler::NOT_RUN )
  759. {
  760. if ( result->m_Status != cmCTestTestHandler::COMPLETED || result->m_ReturnValue )
  761. {
  762. os << "\t\t\t<NamedMeasurement type=\"text/string\" name=\"Exit Code\"><Value>"
  763. << this->GetTestStatus(result->m_Status) << "</Value></NamedMeasurement>\n"
  764. << "\t\t\t<NamedMeasurement type=\"text/string\" name=\"Exit Value\"><Value>"
  765. << result->m_ReturnValue << "</Value></NamedMeasurement>" << std::endl;
  766. }
  767. os << result->m_RegressionImages;
  768. os << "\t\t\t<NamedMeasurement type=\"numeric/double\" "
  769. << "name=\"Execution Time\"><Value>"
  770. << result->m_ExecutionTime << "</Value></NamedMeasurement>\n";
  771. os
  772. << "\t\t\t<NamedMeasurement type=\"text/string\" "
  773. << "name=\"Completion Status\"><Value>"
  774. << result->m_CompletionStatus << "</Value></NamedMeasurement>\n";
  775. }
  776. os
  777. << "\t\t\t<Measurement>\n"
  778. << "\t\t\t\t<Value>";
  779. size_t truncate = result->m_Output.size();
  780. if ( result->m_Status == cmCTestTestHandler::COMPLETED )
  781. {
  782. if ( result->m_Output.size() > m_MaximumPassedTestResultSize )
  783. {
  784. truncate = m_MaximumPassedTestResultSize;
  785. }
  786. }
  787. else
  788. {
  789. if ( result->m_Output.size() > m_MaximumFailedTestResultSize )
  790. {
  791. truncate = m_MaximumFailedTestResultSize;
  792. }
  793. }
  794. os << cmCTest::MakeXMLSafe(result->m_Output.substr(0, truncate));
  795. if ( truncate < result->m_Output.size() )
  796. {
  797. os << "...\n\nThe output was stirpped because it excedes maximum allowed size: "
  798. << truncate << std::endl;
  799. }
  800. os
  801. << "</Value>\n"
  802. << "\t\t\t</Measurement>\n"
  803. << "\t\t</Results>\n"
  804. << "\t</Test>" << std::endl;
  805. }
  806. os << "\t<EndDateTime>" << m_EndTest << "</EndDateTime>\n"
  807. << "<ElapsedMinutes>"
  808. << static_cast<int>(m_ElapsedTestingTime/6)/10.0
  809. << "</ElapsedMinutes>"
  810. << "</Testing>" << std::endl;
  811. m_CTest->EndXML(os);
  812. }
  813. bool cmCTestTestHandler::InitializeMemoryChecking()
  814. {
  815. // Setup the command
  816. if ( cmSystemTools::FileExists(m_CTest->GetDartConfiguration("MemoryCheckCommand").c_str()) )
  817. {
  818. m_MemoryTester
  819. = cmSystemTools::ConvertToOutputPath(m_CTest->GetDartConfiguration("MemoryCheckCommand").c_str());
  820. }
  821. else if ( cmSystemTools::FileExists(m_CTest->GetDartConfiguration("PurifyCommand").c_str()) )
  822. {
  823. m_MemoryTester
  824. = cmSystemTools::ConvertToOutputPath(m_CTest->GetDartConfiguration("PurifyCommand").c_str());
  825. }
  826. else if ( cmSystemTools::FileExists(m_CTest->GetDartConfiguration("ValgrindCommand").c_str()) )
  827. {
  828. m_MemoryTester
  829. = cmSystemTools::ConvertToOutputPath(m_CTest->GetDartConfiguration("ValgrindCommand").c_str());
  830. }
  831. else
  832. {
  833. std::cerr << "Memory checker (MemoryCheckCommand) not set, or cannot find the specified program."
  834. << std::endl;
  835. return false;
  836. }
  837. if ( m_MemoryTester[0] == '\"' && m_MemoryTester[m_MemoryTester.size()-1] == '\"' )
  838. {
  839. m_MemoryTester = m_MemoryTester.substr(1, m_MemoryTester.size()-2);
  840. }
  841. // Setup the options
  842. if ( m_CTest->GetDartConfiguration("MemoryCheckCommandOptions").size() )
  843. {
  844. m_MemoryTesterOptions = m_CTest->GetDartConfiguration("MemoryCheckCommandOptions");
  845. }
  846. else if ( m_CTest->GetDartConfiguration("ValgrindCommandOptions").size() )
  847. {
  848. m_MemoryTesterOptions = m_CTest->GetDartConfiguration("ValgrindCommandOptions");
  849. }
  850. m_MemoryTesterOutputFile = m_CTest->GetToplevelPath() + "/Testing/Temporary/MemoryChecker.log";
  851. m_MemoryTesterOutputFile = cmSystemTools::EscapeSpaces(m_MemoryTesterOutputFile.c_str());
  852. if ( m_MemoryTester.find("valgrind") != std::string::npos )
  853. {
  854. m_MemoryTesterStyle = cmCTestTestHandler::VALGRIND;
  855. if ( !m_MemoryTesterOptions.size() )
  856. {
  857. m_MemoryTesterOptions = "-q --skin=memcheck --leak-check=yes --show-reachable=yes --workaround-gcc296-bugs=yes --num-callers=100";
  858. }
  859. if ( m_CTest->GetDartConfiguration("MemoryCheckSuppressionFile").size() )
  860. {
  861. if ( !cmSystemTools::FileExists(m_CTest->GetDartConfiguration("MemoryCheckSuppressionFile").c_str()) )
  862. {
  863. std::cerr << "Cannot find memory checker suppression file: "
  864. << m_CTest->GetDartConfiguration("MemoryCheckSuppressionFile").c_str() << std::endl;
  865. return false;
  866. }
  867. m_MemoryTesterOptions += " --suppressions=" + cmSystemTools::EscapeSpaces(m_CTest->GetDartConfiguration("MemoryCheckSuppressionFile").c_str()) + "";
  868. }
  869. }
  870. else if ( m_MemoryTester.find("purify") != std::string::npos )
  871. {
  872. m_MemoryTesterStyle = cmCTestTestHandler::PURIFY;
  873. #ifdef _WIN32
  874. m_MemoryTesterOptions += " /SAVETEXTDATA=" + m_MemoryTesterOutputFile;
  875. #else
  876. m_MemoryTesterOptions += " -log-file=" + m_MemoryTesterOutputFile;
  877. #endif
  878. }
  879. else if ( m_MemoryTester.find("boundschecker") != std::string::npos )
  880. {
  881. m_MemoryTesterStyle = cmCTestTestHandler::BOUNDS_CHECKER;
  882. std::cerr << "Bounds checker not yet implemented" << std::endl;
  883. return false;
  884. }
  885. else
  886. {
  887. std::cerr << "Do not understand memory checker: " << m_MemoryTester.c_str() << std::endl;
  888. return false;
  889. }
  890. m_MemoryTesterOptionsParsed = cmSystemTools::ParseArguments(m_MemoryTesterOptions.c_str());
  891. std::vector<cmStdString>::size_type cc;
  892. for ( cc = 0; cmCTestMemCheckResultStrings[cc]; cc ++ )
  893. {
  894. m_MemoryTesterGlobalResults[cc] = 0;
  895. }
  896. return true;
  897. }
  898. int cmCTestTestHandler::ExecuteCommands(std::vector<cmStdString>& vec)
  899. {
  900. std::vector<cmStdString>::iterator it;
  901. for ( it = vec.begin(); it != vec.end(); ++it )
  902. {
  903. int retVal = 0;
  904. if ( m_Verbose )
  905. {
  906. std::cout << "Run command: " << *it << std::endl;
  907. }
  908. if ( !cmSystemTools::RunSingleCommand(it->c_str(), 0, &retVal, 0, true /*m_Verbose*/) ||
  909. retVal != 0 )
  910. {
  911. std::cerr << "Problem running command: " << *it << std::endl;
  912. return 0;
  913. }
  914. }
  915. return 1;
  916. }
  917. std::string cmCTestTestHandler::FindTheExecutable(const char *exe)
  918. {
  919. std::string fullPath = "";
  920. std::string dir;
  921. std::string file;
  922. cmSystemTools::SplitProgramPath(exe, dir, file);
  923. if(m_CTest->GetConfigType() != "" &&
  924. ::TryExecutable(dir.c_str(), file.c_str(), &fullPath,
  925. m_CTest->GetConfigType().c_str()))
  926. {
  927. return fullPath;
  928. }
  929. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,"."))
  930. {
  931. return fullPath;
  932. }
  933. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,""))
  934. {
  935. return fullPath;
  936. }
  937. if ( m_CTest->GetConfigType() == "" )
  938. {
  939. // No config type, so try to guess it
  940. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,"Release"))
  941. {
  942. return fullPath;
  943. }
  944. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,"Debug"))
  945. {
  946. return fullPath;
  947. }
  948. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,"MinSizeRel"))
  949. {
  950. return fullPath;
  951. }
  952. if (::TryExecutable(dir.c_str(),file.c_str(),&fullPath,"RelWithDebInfo"))
  953. {
  954. return fullPath;
  955. }
  956. }
  957. // if everything else failed, check the users path
  958. if (dir != "")
  959. {
  960. std::string path = cmSystemTools::FindProgram(file.c_str());
  961. if (path != "")
  962. {
  963. return path;
  964. }
  965. }
  966. if ( m_CTest->GetConfigType() != "" )
  967. {
  968. dir += "/";
  969. dir += m_CTest->GetConfigType();
  970. dir += "/";
  971. dir += file;
  972. cmSystemTools::Error("config type specified on the command line, but test executable not found.",
  973. dir.c_str());
  974. return "";
  975. }
  976. return fullPath;
  977. }
  978. void cmCTestTestHandler::GetListOfTests(tm_ListOfTests* testlist,
  979. bool memcheck)
  980. {
  981. // does the DartTestfile.txt exist ?
  982. if(!cmSystemTools::FileExists("DartTestfile.txt"))
  983. {
  984. return;
  985. }
  986. // parse the file
  987. std::ifstream fin("DartTestfile.txt");
  988. if(!fin)
  989. {
  990. return;
  991. }
  992. cmsys::RegularExpression ireg(this->m_IncludeRegExp.c_str());
  993. cmsys::RegularExpression ereg(this->m_ExcludeRegExp.c_str());
  994. cmListFileCache cache;
  995. cmListFile* listFile = cache.GetFileCache("DartTestfile.txt", false);
  996. for(std::vector<cmListFileFunction>::const_iterator f =
  997. listFile->m_Functions.begin(); f != listFile->m_Functions.end(); ++f)
  998. {
  999. const cmListFileFunction& lff = *f;
  1000. const std::string& name = lff.m_Name;
  1001. const tm_VectorOfListFileArgs& args = lff.m_Arguments;
  1002. if (name == "SUBDIRS")
  1003. {
  1004. std::string cwd = cmSystemTools::GetCurrentWorkingDirectory();
  1005. for(tm_VectorOfListFileArgs::const_iterator j = args.begin();
  1006. j != args.end(); ++j)
  1007. {
  1008. std::string nwd = cwd + "/";
  1009. nwd += j->Value;
  1010. if (cmSystemTools::FileIsDirectory(nwd.c_str()))
  1011. {
  1012. cmSystemTools::ChangeDirectory(nwd.c_str());
  1013. this->GetListOfTests(testlist, memcheck);
  1014. }
  1015. }
  1016. // return to the original directory
  1017. cmSystemTools::ChangeDirectory(cwd.c_str());
  1018. }
  1019. if (name == "ADD_TEST")
  1020. {
  1021. const std::string& testname = args[0].Value;
  1022. if (this->m_UseExcludeRegExp &&
  1023. this->m_UseExcludeRegExpFirst &&
  1024. ereg.find(testname.c_str()))
  1025. {
  1026. continue;
  1027. }
  1028. if ( memcheck )
  1029. {
  1030. std::vector<cmStdString>::iterator it;
  1031. bool found = false;
  1032. for ( it = m_CustomMemCheckIgnore.begin();
  1033. it != m_CustomMemCheckIgnore.end(); ++ it )
  1034. {
  1035. if ( *it == testname )
  1036. {
  1037. found = true;
  1038. break;
  1039. }
  1040. }
  1041. if ( found )
  1042. {
  1043. if ( m_Verbose )
  1044. {
  1045. std::cout << "Ignore memcheck: " << *it << std::endl;
  1046. }
  1047. continue;
  1048. }
  1049. }
  1050. else
  1051. {
  1052. std::vector<cmStdString>::iterator it;
  1053. bool found = false;
  1054. for ( it = m_CustomTestsIgnore.begin();
  1055. it != m_CustomTestsIgnore.end(); ++ it )
  1056. {
  1057. if ( *it == testname )
  1058. {
  1059. found = true;
  1060. break;
  1061. }
  1062. }
  1063. if ( found )
  1064. {
  1065. if ( m_Verbose )
  1066. {
  1067. std::cout << "Ignore test: " << *it << std::endl;
  1068. }
  1069. continue;
  1070. }
  1071. }
  1072. if (this->m_UseIncludeRegExp && !ireg.find(testname.c_str()))
  1073. {
  1074. continue;
  1075. }
  1076. if (this->m_UseExcludeRegExp &&
  1077. !this->m_UseExcludeRegExpFirst &&
  1078. ereg.find(testname.c_str()))
  1079. {
  1080. continue;
  1081. }
  1082. cmCTestTestProperties test;
  1083. test.m_Name = testname;
  1084. test.m_Args = args;
  1085. test.m_Directory = cmSystemTools::GetCurrentWorkingDirectory();
  1086. testlist->push_back(test);
  1087. }
  1088. }
  1089. }
  1090. void cmCTestTestHandler::UseIncludeRegExp()
  1091. {
  1092. this->m_UseIncludeRegExp = true;
  1093. }
  1094. void cmCTestTestHandler::UseExcludeRegExp()
  1095. {
  1096. this->m_UseExcludeRegExp = true;
  1097. this->m_UseExcludeRegExpFirst = this->m_UseIncludeRegExp ? false : true;
  1098. }
  1099. const char* cmCTestTestHandler::GetTestStatus(int status)
  1100. {
  1101. static const char statuses[][100] = {
  1102. "Not Run",
  1103. "Timeout",
  1104. "SEGFAULT",
  1105. "ILLEGAL",
  1106. "INTERRUPT",
  1107. "NUMERICAL",
  1108. "OTHER_FAULT",
  1109. "Failed",
  1110. "BAD_COMMAND",
  1111. "Completed"
  1112. };
  1113. if ( status < cmCTestTestHandler::NOT_RUN ||
  1114. status > cmCTestTestHandler::COMPLETED )
  1115. {
  1116. return "No Status";
  1117. }
  1118. return statuses[status];
  1119. }
  1120. void cmCTestTestHandler::ExpandTestsToRunInformation(int numTests)
  1121. {
  1122. if (this->TestsToRunString.empty())
  1123. {
  1124. return;
  1125. }
  1126. int start;
  1127. int end = -1;
  1128. double stride = -1;
  1129. std::string::size_type pos = 0;
  1130. std::string::size_type pos2;
  1131. // read start
  1132. if(GetNextNumber(this->TestsToRunString, start, pos, pos2))
  1133. {
  1134. // read end
  1135. if(GetNextNumber(this->TestsToRunString, end, pos, pos2))
  1136. {
  1137. // read stride
  1138. if(GetNextRealNumber(this->TestsToRunString, stride, pos, pos2))
  1139. {
  1140. int val =0;
  1141. // now read specific numbers
  1142. while(GetNextNumber(this->TestsToRunString, val, pos, pos2))
  1143. {
  1144. m_TestsToRun.push_back(val);
  1145. }
  1146. m_TestsToRun.push_back(val);
  1147. }
  1148. }
  1149. }
  1150. // if start is not specified then we assume we start at 1
  1151. if(start == -1)
  1152. {
  1153. start = 1;
  1154. }
  1155. // if end isnot specified then we assume we end with the last test
  1156. if(end == -1)
  1157. {
  1158. end = numTests;
  1159. }
  1160. // if the stride wasn't specified then it defaults to 1
  1161. if(stride == -1)
  1162. {
  1163. stride = 1;
  1164. }
  1165. // if we have a range then add it
  1166. if(end != -1 && start != -1 && stride > 0)
  1167. {
  1168. int i = 0;
  1169. while (i*stride + start <= end)
  1170. {
  1171. m_TestsToRun.push_back(static_cast<int>(i*stride+start));
  1172. ++i;
  1173. }
  1174. }
  1175. // sort the array
  1176. std::sort(m_TestsToRun.begin(), m_TestsToRun.end(), std::less<int>());
  1177. // remove duplicates
  1178. std::vector<int>::iterator new_end =
  1179. std::unique(m_TestsToRun.begin(), m_TestsToRun.end());
  1180. m_TestsToRun.erase(new_end, m_TestsToRun.end());
  1181. std::cout << "Running tests: ";
  1182. for(unsigned int i =0; i < m_TestsToRun.size(); ++i)
  1183. {
  1184. std::cout << m_TestsToRun[i] << " ";
  1185. }
  1186. std::cout << "\n";
  1187. }
  1188. #define SPACE_REGEX "[ \t\r\n]"
  1189. std::string cmCTestTestHandler::GenerateRegressionImages(
  1190. const std::string& xml)
  1191. {
  1192. cmsys::RegularExpression twoattributes(
  1193. "<DartMeasurement"
  1194. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1195. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1196. SPACE_REGEX "*>([^<]*)</DartMeasurement>");
  1197. cmsys::RegularExpression threeattributes(
  1198. "<DartMeasurement"
  1199. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1200. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1201. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1202. SPACE_REGEX "*>([^<]*)</DartMeasurement>");
  1203. cmsys::RegularExpression fourattributes(
  1204. "<DartMeasurement"
  1205. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1206. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1207. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1208. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1209. SPACE_REGEX "*>([^<]*)</DartMeasurement>");
  1210. cmsys::RegularExpression measurementfile(
  1211. "<DartMeasurementFile"
  1212. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1213. SPACE_REGEX "*(name|type|encoding|compression)=\"([^\"]*)\""
  1214. SPACE_REGEX "*>([^<]*)</DartMeasurementFile>");
  1215. cmOStringStream ostr;
  1216. bool done = false;
  1217. std::string cxml = xml;
  1218. while ( ! done )
  1219. {
  1220. if ( twoattributes.find(cxml) )
  1221. {
  1222. ostr
  1223. << "\t\t\t<NamedMeasurement"
  1224. << " " << twoattributes.match(1) << "=\"" << twoattributes.match(2) << "\""
  1225. << " " << twoattributes.match(3) << "=\"" << twoattributes.match(4) << "\""
  1226. << "><Value>" << twoattributes.match(5)
  1227. << "</Value></NamedMeasurement>"
  1228. << std::endl;
  1229. cxml.erase(twoattributes.start(), twoattributes.end() - twoattributes.start());
  1230. }
  1231. else if ( threeattributes.find(cxml) )
  1232. {
  1233. ostr
  1234. << "\t\t\t<NamedMeasurement"
  1235. << " " << threeattributes.match(1) << "=\"" << threeattributes.match(2) << "\""
  1236. << " " << threeattributes.match(3) << "=\"" << threeattributes.match(4) << "\""
  1237. << " " << threeattributes.match(5) << "=\"" << threeattributes.match(6) << "\""
  1238. << "><Value>" << threeattributes.match(7)
  1239. << "</Value></NamedMeasurement>"
  1240. << std::endl;
  1241. cxml.erase(threeattributes.start(), threeattributes.end() - threeattributes.start());
  1242. }
  1243. else if ( fourattributes.find(cxml) )
  1244. {
  1245. ostr
  1246. << "\t\t\t<NamedMeasurement"
  1247. << " " << fourattributes.match(1) << "=\"" << fourattributes.match(2) << "\""
  1248. << " " << fourattributes.match(3) << "=\"" << fourattributes.match(4) << "\""
  1249. << " " << fourattributes.match(5) << "=\"" << fourattributes.match(6) << "\""
  1250. << " " << fourattributes.match(7) << "=\"" << fourattributes.match(8) << "\""
  1251. << "><Value>" << fourattributes.match(9)
  1252. << "</Value></NamedMeasurement>"
  1253. << std::endl;
  1254. cxml.erase(fourattributes.start(), fourattributes.end() - fourattributes.start());
  1255. }
  1256. else if ( measurementfile.find(cxml) )
  1257. {
  1258. const std::string& filename =
  1259. cmCTest::CleanString(measurementfile.match(5));
  1260. if ( cmSystemTools::FileExists(filename.c_str()) )
  1261. {
  1262. long len = cmSystemTools::FileLength(filename.c_str());
  1263. if ( len == 0 )
  1264. {
  1265. std::string k1 = measurementfile.match(1);
  1266. std::string v1 = measurementfile.match(2);
  1267. std::string k2 = measurementfile.match(3);
  1268. std::string v2 = measurementfile.match(4);
  1269. if ( cmSystemTools::LowerCase(k1) == "type" )
  1270. {
  1271. v1 = "text/string";
  1272. }
  1273. if ( cmSystemTools::LowerCase(k2) == "type" )
  1274. {
  1275. v2 = "text/string";
  1276. }
  1277. ostr
  1278. << "\t\t\t<NamedMeasurement"
  1279. << " " << k1 << "=\"" << v1 << "\""
  1280. << " " << k2 << "=\"" << v2 << "\""
  1281. << " encoding=\"none\""
  1282. << "><Value>Image " << filename.c_str()
  1283. << " is empty</Value></NamedMeasurement>";
  1284. }
  1285. else
  1286. {
  1287. std::ifstream ifs(filename.c_str(), std::ios::in
  1288. #ifdef _WIN32
  1289. | std::ios::binary
  1290. #endif
  1291. );
  1292. unsigned char *file_buffer = new unsigned char [ len + 1 ];
  1293. ifs.read(reinterpret_cast<char*>(file_buffer), len);
  1294. unsigned char *encoded_buffer = new unsigned char [ static_cast<int>(len * 1.5 + 5) ];
  1295. unsigned long rlen = cmsysBase64_Encode(file_buffer, len, encoded_buffer, 1);
  1296. unsigned long cc;
  1297. ostr
  1298. << "\t\t\t<NamedMeasurement"
  1299. << " " << measurementfile.match(1) << "=\"" << measurementfile.match(2) << "\""
  1300. << " " << measurementfile.match(3) << "=\"" << measurementfile.match(4) << "\""
  1301. << " encoding=\"base64\""
  1302. << ">" << std::endl << "\t\t\t\t<Value>";
  1303. for ( cc = 0; cc < rlen; cc ++ )
  1304. {
  1305. ostr << encoded_buffer[cc];
  1306. if ( cc % 60 == 0 && cc )
  1307. {
  1308. ostr << std::endl;
  1309. }
  1310. }
  1311. ostr
  1312. << "</Value>" << std::endl << "\t\t\t</NamedMeasurement>"
  1313. << std::endl;
  1314. delete [] file_buffer;
  1315. delete [] encoded_buffer;
  1316. }
  1317. }
  1318. else
  1319. {
  1320. int idx = 4;
  1321. if ( measurementfile.match(1) == "name" )
  1322. {
  1323. idx = 2;
  1324. }
  1325. ostr
  1326. << "\t\t\t<NamedMeasurement"
  1327. << " name=\"" << measurementfile.match(idx) << "\""
  1328. << " text=\"text/string\""
  1329. << "><Value>File " << filename.c_str() << " not found</Value></NamedMeasurement>"
  1330. << std::endl;
  1331. std::cout << "File \"" << filename.c_str() << "\" not found." << std::endl;
  1332. }
  1333. cxml.erase(measurementfile.start(), measurementfile.end() - measurementfile.start());
  1334. }
  1335. else
  1336. {
  1337. done = true;
  1338. }
  1339. }
  1340. return ostr.str();
  1341. }
  1342. bool cmCTestTestHandler::ProcessMemCheckOutput(const std::string& str,
  1343. std::string& log, int* results)
  1344. {
  1345. std::string::size_type cc;
  1346. for ( cc = 0; cc < cmCTestTestHandler::NO_MEMORY_FAULT; cc ++ )
  1347. {
  1348. results[cc] = 0;
  1349. }
  1350. if ( m_MemoryTesterStyle == cmCTestTestHandler::VALGRIND )
  1351. {
  1352. return this->ProcessMemCheckValgrindOutput(str, log, results);
  1353. }
  1354. else if ( m_MemoryTesterStyle == cmCTestTestHandler::PURIFY )
  1355. {
  1356. return this->ProcessMemCheckPurifyOutput(str, log, results);
  1357. }
  1358. else if ( m_MemoryTesterStyle == cmCTestTestHandler::BOUNDS_CHECKER )
  1359. {
  1360. log.append("\nMemory checking style used was: ");
  1361. log.append("Bounds Checker");
  1362. }
  1363. else
  1364. {
  1365. log.append("\nMemory checking style used was: ");
  1366. log.append("None that I know");
  1367. log = str;
  1368. }
  1369. return true;
  1370. }
  1371. bool cmCTestTestHandler::ProcessMemCheckPurifyOutput(
  1372. const std::string&, std::string& log,
  1373. int* results)
  1374. {
  1375. if ( !cmSystemTools::FileExists(m_MemoryTesterOutputFile.c_str()) )
  1376. {
  1377. log = "Cannot find Purify output file: " + m_MemoryTesterOutputFile;
  1378. std::cerr << log.c_str() << std::endl;
  1379. return false;
  1380. }
  1381. std::ifstream ifs(m_MemoryTesterOutputFile.c_str());
  1382. if ( !ifs )
  1383. {
  1384. log = "Cannot read Purify output file: " + m_MemoryTesterOutputFile;
  1385. std::cerr << log.c_str() << std::endl;
  1386. return false;
  1387. }
  1388. cmOStringStream ostr;
  1389. log = "";
  1390. cmsys::RegularExpression pfW("^\\[[WEI]\\] ([A-Z][A-Z][A-Z][A-Z]*): ");
  1391. int defects = 0;
  1392. std::string line;
  1393. while ( cmSystemTools::GetLineFromStream(ifs, line) )
  1394. {
  1395. int failure = cmCTestTestHandler::NO_MEMORY_FAULT;
  1396. if ( pfW.find(line) )
  1397. {
  1398. int cc;
  1399. for ( cc = 0; cc < cmCTestTestHandler::NO_MEMORY_FAULT; cc ++ )
  1400. {
  1401. if ( pfW.match(1) == cmCTestMemCheckResultStrings[cc] )
  1402. {
  1403. failure = cc;
  1404. break;
  1405. }
  1406. }
  1407. if ( cc == cmCTestTestHandler::NO_MEMORY_FAULT )
  1408. {
  1409. std::cerr<< "Unknown Purify memory fault: " << pfW.match(1) << std::endl;
  1410. ostr << "*** Unknown Purify memory fault: " << pfW.match(1) << std::endl;
  1411. }
  1412. }
  1413. if ( failure != NO_MEMORY_FAULT )
  1414. {
  1415. ostr << "<b>" << cmCTestMemCheckResultStrings[failure] << "</b> ";
  1416. results[failure] ++;
  1417. defects ++;
  1418. }
  1419. ostr << cmCTest::MakeXMLSafe(line) << std::endl;
  1420. }
  1421. log = ostr.str();
  1422. if ( defects )
  1423. {
  1424. return false;
  1425. }
  1426. return true;
  1427. }
  1428. bool cmCTestTestHandler::ProcessMemCheckValgrindOutput(
  1429. const std::string& str, std::string& log,
  1430. int* results)
  1431. {
  1432. std::vector<cmStdString> lines;
  1433. cmSystemTools::Split(str.c_str(), lines);
  1434. std::string::size_type cc;
  1435. cmOStringStream ostr;
  1436. log = "";
  1437. int defects = 0;
  1438. cmsys::RegularExpression valgrindLine("^==[0-9][0-9]*==");
  1439. cmsys::RegularExpression vgFIM(
  1440. "== .*Invalid free\\(\\) / delete / delete\\[\\]");
  1441. cmsys::RegularExpression vgFMM(
  1442. "== .*Mismatched free\\(\\) / delete / delete \\[\\]");
  1443. cmsys::RegularExpression vgMLK(
  1444. "== .*[0-9][0-9]* bytes in [0-9][0-9]* blocks are definitely lost"
  1445. " in loss record [0-9][0-9]* of [0-9]");
  1446. cmsys::RegularExpression vgPAR(
  1447. "== .*Syscall param .* contains unaddressable byte\\(s\\)");
  1448. cmsys::RegularExpression vgMPK1(
  1449. "== .*[0-9][0-9]* bytes in [0-9][0-9]* blocks are possibly lost in"
  1450. " loss record [0-9][0-9]* of [0-9]");
  1451. cmsys::RegularExpression vgMPK2(
  1452. "== .*[0-9][0-9]* bytes in [0-9][0-9]* blocks are still reachable"
  1453. " in loss record [0-9][0-9]* of [0-9]");
  1454. cmsys::RegularExpression vgUMC(
  1455. "== .*Conditional jump or move depends on uninitialised value\\(s\\)");
  1456. cmsys::RegularExpression vgUMR1("== .*Use of uninitialised value of size [0-9][0-9]*");
  1457. cmsys::RegularExpression vgUMR2("== .*Invalid read of size [0-9][0-9]*");
  1458. cmsys::RegularExpression vgUMR3("== .*Jump to the invalid address ");
  1459. cmsys::RegularExpression vgUMR4(
  1460. "== .*Syscall param .* contains uninitialised or unaddressable byte\\(s\\)");
  1461. cmsys::RegularExpression vgIPW("== .*Invalid write of size [0-9]");
  1462. cmsys::RegularExpression vgABR("== .*pthread_mutex_unlock: mutex is locked by a different thread");
  1463. //double sttime = cmSystemTools::GetTime();
  1464. //std::cout << "Start test: " << lines.size() << std::endl;
  1465. for ( cc = 0; cc < lines.size(); cc ++ )
  1466. {
  1467. if ( valgrindLine.find(lines[cc]) )
  1468. {
  1469. int failure = cmCTestTestHandler::NO_MEMORY_FAULT;
  1470. if ( vgFIM.find(lines[cc]) ) { failure = cmCTestTestHandler::FIM; }
  1471. else if ( vgFMM.find(lines[cc]) ) { failure = cmCTestTestHandler::FMM; }
  1472. else if ( vgMLK.find(lines[cc]) ) { failure = cmCTestTestHandler::MLK; }
  1473. else if ( vgPAR.find(lines[cc]) ) { failure = cmCTestTestHandler::PAR; }
  1474. else if ( vgMPK1.find(lines[cc]) ){ failure = cmCTestTestHandler::MPK; }
  1475. else if ( vgMPK2.find(lines[cc]) ){ failure = cmCTestTestHandler::MPK; }
  1476. else if ( vgUMC.find(lines[cc]) ) { failure = cmCTestTestHandler::UMC; }
  1477. else if ( vgUMR1.find(lines[cc]) ){ failure = cmCTestTestHandler::UMR; }
  1478. else if ( vgUMR2.find(lines[cc]) ){ failure = cmCTestTestHandler::UMR; }
  1479. else if ( vgUMR3.find(lines[cc]) ){ failure = cmCTestTestHandler::UMR; }
  1480. else if ( vgUMR4.find(lines[cc]) ){ failure = cmCTestTestHandler::UMR; }
  1481. else if ( vgIPW.find(lines[cc]) ) { failure = cmCTestTestHandler::IPW; }
  1482. else if ( vgABR.find(lines[cc]) ) { failure = cmCTestTestHandler::ABR; }
  1483. if ( failure != cmCTestTestHandler::NO_MEMORY_FAULT )
  1484. {
  1485. ostr << "<b>" << cmCTestMemCheckResultStrings[failure] << "</b> ";
  1486. results[failure] ++;
  1487. defects ++;
  1488. }
  1489. ostr << cmCTest::MakeXMLSafe(lines[cc]) << std::endl;
  1490. }
  1491. }
  1492. //std::cout << "End test (elapsed: " << (cmSystemTools::GetTime() - sttime) << std::endl;
  1493. log = ostr.str();
  1494. if ( defects )
  1495. {
  1496. return false;
  1497. }
  1498. return true;
  1499. }
  1500. void cmCTestTestHandler::SetIncludeRegExp(const char *arg)
  1501. {
  1502. m_IncludeRegExp = arg;
  1503. }
  1504. void cmCTestTestHandler::SetExcludeRegExp(const char *arg)
  1505. {
  1506. m_ExcludeRegExp = arg;
  1507. }
  1508. void cmCTestTestHandler::SetTestsToRunInformation(const char* in)
  1509. {
  1510. this->TestsToRunString = in;
  1511. // if the argument is a file, then read it and use the contents as the string
  1512. if(cmSystemTools::FileExists(in))
  1513. {
  1514. std::ifstream fin(in);
  1515. unsigned long filelen = cmSystemTools::FileLength(in);
  1516. char* buff = new char[filelen+1];
  1517. fin.getline(buff, filelen);
  1518. buff[fin.gcount()] = 0;
  1519. this->TestsToRunString = buff;
  1520. }
  1521. }