C++ Utilities  4.17.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/nativefilestream.h"
10 #include "../io/path.h"
11 
12 #include <cerrno>
13 #include <cstdlib>
14 #include <cstring>
15 #include <fstream>
16 #include <initializer_list>
17 #include <iostream>
18 #include <limits>
19 
20 #ifdef PLATFORM_UNIX
21 #include <poll.h>
22 #include <sys/stat.h>
23 #include <sys/wait.h>
24 #include <unistd.h>
25 #endif
26 
27 #ifdef PLATFORM_WINDOWS
28 #include <windows.h>
29 #endif
30 
31 using namespace std;
32 using namespace ApplicationUtilities;
33 using namespace ConversionUtilities;
34 using namespace EscapeCodes;
35 using namespace IoUtilities;
36 
40 namespace TestUtilities {
41 
42 bool fileSystemItemExists(const string &path)
43 {
44 #ifdef PLATFORM_UNIX
45  struct stat res;
46  return stat(path.data(), &res) == 0;
47 #else
48  const auto widePath(convertMultiByteToWide(path));
49  if (!widePath.first) {
50  return false;
51  }
52  const auto fileType(GetFileAttributesW(widePath.first.get()));
53  return fileType != INVALID_FILE_ATTRIBUTES;
54 #endif
55 }
56 
57 bool fileExists(const string &path)
58 {
59 #ifdef PLATFORM_UNIX
60  struct stat res;
61  return stat(path.data(), &res) == 0 && !S_ISDIR(res.st_mode);
62 #else
63  const auto widePath(convertMultiByteToWide(path));
64  if (!widePath.first) {
65  return false;
66  }
67  const auto fileType(GetFileAttributesW(widePath.first.get()));
68  return (fileType != INVALID_FILE_ATTRIBUTES) && !(fileType & FILE_ATTRIBUTE_DIRECTORY) && !(fileType & FILE_ATTRIBUTE_DEVICE);
69 #endif
70 }
71 
72 bool dirExists(const string &path)
73 {
74 #ifdef PLATFORM_UNIX
75  struct stat res;
76  return stat(path.data(), &res) == 0 && S_ISDIR(res.st_mode);
77 #else
78  const auto widePath(convertMultiByteToWide(path));
79  if (!widePath.first) {
80  return false;
81  }
82  const auto fileType(GetFileAttributesW(widePath.first.get()));
83  return (fileType != INVALID_FILE_ATTRIBUTES) && (fileType & FILE_ATTRIBUTE_DIRECTORY);
84 #endif
85 }
86 
87 bool makeDir(const string &path)
88 {
89 #ifdef PLATFORM_UNIX
90  return mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0;
91 #else
92  const auto widePath(convertMultiByteToWide(path));
93  if (!widePath.first) {
94  return false;
95  }
96  return CreateDirectoryW(widePath.first.get(), nullptr) || GetLastError() == ERROR_ALREADY_EXISTS;
97 #endif
98 }
99 
100 TestApplication *TestApplication::m_instance = nullptr;
101 
112 TestApplication::TestApplication(int argc, char **argv)
113  : m_helpArg(m_parser)
114  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
115  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
116  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
117  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
118 {
119  // check whether there is already an instance
120  if (m_instance) {
121  throw runtime_error("only one TestApplication instance allowed at a time");
122  }
123  m_instance = this;
124 
125  // determine fallback path for testfiles which is used when --test-files-path/-p not present
126  // -> read TEST_FILE_PATH environment variable
127  m_fallbackTestFilesPath = readTestfilePathFromEnv();
128  // -> find source directory if TEST_FILE_PATH not present
129  bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
130  if (fallbackIsSourceDir) {
131  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
132  }
133 
134  // handle specified arguments (if present)
135  if (argc && argv) {
136  // setup argument parser
137  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
138  arg->setRequiredValueCount(1);
139  arg->setValueNames({ "path" });
140  arg->setCombinable(true);
141  }
142  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
143  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
144  m_unitsArg.setCombinable(true);
145  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
146 
147  // parse arguments
148  try {
149  m_parser.parseArgs(argc, argv);
150  } catch (const Failure &failure) {
151  cerr << failure;
152  m_valid = false;
153  return;
154  }
155 
156  // print help
157  if (m_helpArg.isPresent()) {
158  exit(0);
159  }
160  }
161 
162  // handle path for testfiles and working-copy
163  cerr << "Directories used to search for testfiles:" << endl;
164  if (m_testFilesPathArg.isPresent()) {
165  if (*m_testFilesPathArg.values().front()) {
166  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
167  } else {
168  cerr << (m_testFilesPath = "./") << endl;
169  }
170  } else {
171  // use fallback path if --test-files-path/-p not present
172  m_testFilesPath.swap(m_fallbackTestFilesPath);
173  cerr << m_testFilesPath << endl;
174  }
175  // if it wasn't already the case, use the source directory as fallback dir
176  if (m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
177  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
178  fallbackIsSourceDir = true;
179  }
180  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
181  cerr << m_fallbackTestFilesPath << endl;
182  }
183  cerr << "./testfiles/" << endl << endl;
184  cerr << "Directory used to store working copies:" << endl;
185  if (m_workingDirArg.isPresent()) {
186  if (*m_workingDirArg.values().front()) {
187  (m_workingDir = m_workingDirArg.values().front()) += '/';
188  } else {
189  m_workingDir = "./";
190  }
191  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
192  if (*workingDirEnv) {
193  m_workingDir = argsToString(workingDirEnv, '/');
194  }
195  } else {
196  if (m_testFilesPathArg.isPresent()) {
197  m_workingDir = m_testFilesPath + "workingdir/";
198  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
199  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
200  } else {
201  m_workingDir = "./testfiles/workingdir/";
202  }
203  }
204  cerr << m_workingDir << endl << endl;
205 
206  // clear list of all additional profiling files created when forking the test application
207  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
208  ofstream(profrawListFile, ios_base::trunc);
209  }
210 
211  m_valid = true;
212  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
213 }
214 
219 {
220  m_instance = nullptr;
221 }
222 
236 string TestApplication::testFilePath(const string &relativeTestFilePath) const
237 {
238  string path;
239 
240  // check the path specified by command line argument or via environment variable
241  if (!m_testFilesPath.empty()) {
242  if (fileExists(path = m_testFilesPath + relativeTestFilePath)) {
243  return path;
244  }
245  }
246 
247  // check the fallback path (value from environment variable or source directory)
248  if (!m_fallbackTestFilesPath.empty()) {
249  if (fileExists(path = m_fallbackTestFilesPath + relativeTestFilePath)) {
250  return path;
251  }
252  }
253 
254  // file still not found -> return default path
255  if (!fileExists(path = "./testfiles/" + relativeTestFilePath)) {
256  cerr << Phrases::Warning << "The testfile \"" << relativeTestFilePath << "\" can not be located." << Phrases::EndFlush;
257  }
258  return path;
259 }
260 
268 string TestApplication::workingCopyPathMode(const string &relativeTestFilePath, WorkingCopyMode mode) const
269 {
270  return workingCopyPathAs(relativeTestFilePath, relativeTestFilePath, mode);
271 }
272 
277 string TestApplication::workingCopyPath(const string &relativeTestFilePath) const
278 {
279  return workingCopyPathAs(relativeTestFilePath, relativeTestFilePath, WorkingCopyMode::CreateCopy);
280 }
281 
297  const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode) const
298 {
299  // ensure working directory is present
300  if (!dirExists(m_workingDir) && !makeDir(m_workingDir)) {
301  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": can't create working directory \""
302  << m_workingDir << "\"." << Phrases::EndFlush;
303  return string();
304  }
305 
306  // ensure subdirectory exists
307  const auto parts = splitString<vector<string>>(relativeWorkingCopyPath, "/", EmptyPartsTreat::Omit);
308  if (!parts.empty()) {
309  // create subdirectory level by level
310  string currentLevel;
311  currentLevel.reserve(m_workingDir.size() + relativeWorkingCopyPath.size() + 1);
312  currentLevel.assign(m_workingDir);
313  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
314  if (currentLevel.back() != '/') {
315  currentLevel += '/';
316  }
317  currentLevel += *i;
318 
319  // continue if subdirectory level already exists or we can successfully create the directory
320  if (dirExists(currentLevel) || makeDir(currentLevel)) {
321  continue;
322  }
323  // fail otherwise
324  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeWorkingCopyPath << "\": can't create directory \""
325  << currentLevel << "\" (inside working directory)." << Phrases::EndFlush;
326  return string();
327  }
328  }
329 
330  // just return the path if we don't want to actually create a copy
331  if (mode == WorkingCopyMode::NoCopy) {
332  return m_workingDir + relativeWorkingCopyPath;
333  }
334 
335  // copy the file
336  const auto origFilePath(testFilePath(relativeTestFilePath));
337  auto workingCopyPath(m_workingDir + relativeWorkingCopyPath);
338  size_t workingCopyPathAttempt = 0;
339  NativeFileStream origFile, workingCopy;
340  origFile.open(origFilePath, ios_base::in | ios_base::binary);
341  if (origFile.fail()) {
342  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
343  << "\": an IO error occurred when opening original file \"" << origFilePath << "\"." << Phrases::EndFlush;
344  cerr << "error: " << strerror(errno) << endl;
345  return string();
346  }
347  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
348  while (workingCopy.fail() && fileSystemItemExists(workingCopyPath)) {
349  // adjust the working copy path if the target file already exists and can not be truncated
350  workingCopyPath = argsToString(m_workingDir, relativeWorkingCopyPath, '.', ++workingCopyPathAttempt);
351  workingCopy.clear();
352  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
353  }
354  if (workingCopy.fail()) {
355  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
356  << "\": an IO error occurred when opening target file \"" << workingCopyPath << "\"." << Phrases::EndFlush;
357  cerr << "error: " << strerror(errno) << endl;
358  return string();
359  }
360  workingCopy << origFile.rdbuf();
361  if (!origFile.fail() && !workingCopy.fail()) {
362  return workingCopyPath;
363  }
364 
365  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": ";
366  if (origFile.fail()) {
367  cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
368  return string();
369  }
370  if (workingCopy.fail()) {
371  if (origFile.fail()) {
372  cerr << " and ";
373  }
374  cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
375  }
376  cerr << "error: " << strerror(errno) << endl;
377  return string();
378 }
379 
380 #ifdef PLATFORM_UNIX
381 
385 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
386  const std::string &newProfilingPath)
387 {
388  // print log message
389  if (!suppressLogging) {
390  // print actual appPath and skip first argument instead
391  cout << '-' << ' ' << appPath;
392  if (*args) {
393  for (const char *const *i = args + 1; *i; ++i) {
394  cout << ' ' << *i;
395  }
396  }
397  cout << endl;
398  }
399 
400  // create pipes
401  int coutPipes[2], cerrPipes[2];
402  pipe(coutPipes);
403  pipe(cerrPipes);
404  const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
405  const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
406 
407  // create child process
408  if (const auto child = fork()) {
409  // parent process: read stdout and stderr from child
410  close(writeCoutPipe);
411  close(writeCerrPipe);
412 
413  try {
414  if (child == -1) {
415  throw runtime_error("Unable to create fork");
416  }
417 
418  // init file descriptor set for poll
419  struct pollfd fileDescriptorSet[2];
420  fileDescriptorSet[0].fd = readCoutPipe;
421  fileDescriptorSet[1].fd = readCerrPipe;
422  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
423 
424  // init variables for reading
425  char buffer[512];
426  output.clear();
427  errors.clear();
428 
429  // poll as long as at least one pipe is open
430  do {
431  const auto retpoll = poll(fileDescriptorSet, 2, timeout);
432  if (retpoll == 0) {
433  throw runtime_error("Poll time-out");
434  }
435  if (retpoll < 0) {
436  throw runtime_error("Poll failed");
437  }
438  if (fileDescriptorSet[0].revents & POLLIN) {
439  const auto count = read(readCoutPipe, buffer, sizeof(buffer));
440  if (count > 0) {
441  output.append(buffer, static_cast<size_t>(count));
442  }
443  } else if (fileDescriptorSet[0].revents & POLLHUP) {
444  close(readCoutPipe);
445  fileDescriptorSet[0].fd = -1;
446  }
447  if (fileDescriptorSet[1].revents & POLLIN) {
448  const auto count = read(readCerrPipe, buffer, sizeof(buffer));
449  if (count > 0) {
450  errors.append(buffer, static_cast<size_t>(count));
451  }
452  } else if (fileDescriptorSet[1].revents & POLLHUP) {
453  close(readCerrPipe);
454  fileDescriptorSet[1].fd = -1;
455  }
456  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
457  } catch (...) {
458  // ensure all pipes are closed in the error case
459  close(readCoutPipe);
460  close(readCerrPipe);
461  throw;
462  }
463 
464  // get return code
465  int childReturnCode;
466  waitpid(child, &childReturnCode, 0);
467  return childReturnCode;
468  } else {
469  // child process
470  // -> set pipes to be used for stdout/stderr
471  dup2(writeCoutPipe, STDOUT_FILENO);
472  dup2(writeCerrPipe, STDERR_FILENO);
473  close(readCoutPipe);
474  close(writeCoutPipe);
475  close(readCerrPipe);
476  close(writeCerrPipe);
477 
478  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
479  if (!newProfilingPath.empty()) {
480  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
481  }
482 
483  // -> execute application
484  execv(appPath, const_cast<char *const *>(args));
485  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
486  exit(-101);
487  }
488 }
489 
499 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
500 {
501  // increase counter used for giving profiling files unique names
502  static unsigned int invocationCount = 0;
503  ++invocationCount;
504 
505  // determine the path of the application to be tested
506  const char *appPath = m_applicationPathArg.firstValue();
507  string fallbackAppPath;
508  if (!appPath || !*appPath) {
509  // try to find the path by removing "_tests"-suffix from own executable path
510  // (the own executable path is the path of the test application and its name is usually the name of the application
511  // to be tested with "_tests"-suffix)
512  const char *const testAppPath = m_parser.executable();
513  const size_t testAppPathLength = strlen(testAppPath);
514  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
515  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
516  appPath = fallbackAppPath.data();
517  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
518  } else {
519  throw runtime_error("Unable to execute application to be tested: no application path specified");
520  }
521  }
522 
523  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
524  string newProfilingPath;
525  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
526  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
527  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
528  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
529  // extract application name from path
530  const char *appName = strrchr(appPath, '/');
531  appName = appName ? appName + 1 : appPath;
532  // concat new path
533  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
534  // append path to profiling list file
535  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
536  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
537  }
538  }
539  }
540 
541  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
542 }
543 
551 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
552 {
553  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
554 }
555 #endif // PLATFORM_UNIX
556 
557 string TestApplication::readTestfilePathFromEnv()
558 {
559  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
560  if (!testFilesPathEnv || !*testFilesPathEnv) {
561  return string();
562  }
563  return argsToString(testFilesPathEnv, '/');
564 }
565 
566 string TestApplication::readTestfilePathFromSrcRef()
567 {
568  try {
569  // read "srcdirref" file which should contain the path of the source directory; this file should have been
570  // create by the CMake module "TestTarget.cmake"
571  auto srcDirContent(readFile("srcdirref", 2 * 1024));
572  if (srcDirContent.empty()) {
573  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
574  return string();
575  }
576 
577  // check whether the referenced source directory contains a "testfiles" directory
578 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
579  bool hasTestfilesDir = false;
580  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
581  if (dir == "testfiles") {
582  hasTestfilesDir = true;
583  break;
584  }
585  }
586  if (!hasTestfilesDir) {
587  cerr << Phrases::Warning
588  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
589  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
590  return string();
591  }
592 #endif // PLATFORM_UNIX
593 
594  return srcDirContent += "/testfiles/";
595  } catch (...) {
596  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
597  << Phrases::EndFlush;
598  catchIoFailure();
599  }
600  return string();
601 }
602 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types.
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
bool fileSystemItemExists(const string &path)
Definition: testutils.cpp:42
std::string workingCopyPath(const std::string &relativeTestFilePath) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:277
bool makeDir(const string &path)
Definition: testutils.cpp:87
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
std::fstream NativeFileStream
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
bool fileExists(const string &path)
Definition: testutils.cpp:57
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:81
std::string testFilePath(const std::string &relativeTestFilePath) const
Returns the full path of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:236
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:13
constexpr int i
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:218
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
std::string workingCopyPathMode(const std::string &relativeTestFilePath, WorkingCopyMode mode) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:268
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
void setValueNames(std::initializer_list< const char * > valueNames)
Sets the names of the requried values.
bool dirExists(const string &path)
Definition: testutils.cpp:72
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:18
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
std::string workingCopyPathAs(const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode=WorkingCopyMode::CreateCopy) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:296