From 906749a48e3ead4dda1e61378759c11267451fe0 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 12 Feb 2018 18:11:01 -0700 Subject: [PATCH 001/330] Work around Gtest build failure caused by -Werror=unused-function. (#529) We're propagating extra warning flags to the gtest build, which can cause it to fail. This patch prevents passing "-Wextra" to gtest, since the library itself doesn't test with that flag. --- cmake/HandleGTest.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 77ffc4c51c..9123babfd3 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -23,6 +23,11 @@ macro(build_external_gtest) if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE") set(GTEST_BUILD_TYPE "DEBUG") endif() + # FIXME: Since 10/Feb/2017 the googletest trunk has had a bug where + # -Werror=unused-function fires during the build on OS X. This is a temporary + # workaround to keep our travis bots from failing. It should be removed + # once gtest is fixed. + list(APPEND GTEST_FLAGS "-Wno-unused-function") split_list(GTEST_FLAGS) ExternalProject_Add(googletest EXCLUDE_FROM_ALL ON From 562f9d256d30fdd2a4200109e0f79ba57d05f488 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 12 Feb 2018 18:43:32 -0700 Subject: [PATCH 002/330] Fix GTest workaround on MSVC --- cmake/HandleGTest.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 9123babfd3..3cf8e18b2a 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -27,7 +27,9 @@ macro(build_external_gtest) # -Werror=unused-function fires during the build on OS X. This is a temporary # workaround to keep our travis bots from failing. It should be removed # once gtest is fixed. - list(APPEND GTEST_FLAGS "-Wno-unused-function") + if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + list(APPEND GTEST_FLAGS "-Wno-unused-function") + endif() split_list(GTEST_FLAGS) ExternalProject_Add(googletest EXCLUDE_FROM_ALL ON From dd8dcc8da1fb03cced6129041e3f06a607889346 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 12 Feb 2018 19:07:19 -0700 Subject: [PATCH 003/330] Make output tests more stable on slow machines. The appveyor bot sometimes fails because the time it outputs is 6 digits long, but the output test regex expects at most 5 digits. This patch increases the size to 6 digits to placate the test. This should not *really* affect the correctness of the test. --- test/output_test_helper.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 24746f6d27..6b18fe4359 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -40,8 +40,8 @@ SubMap& GetSubstitutions() { {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, - {"%time", "[ ]*[0-9]{1,5} ns"}, - {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, + {"%time", "[ ]*[0-9]{1,6} ns"}, + {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"}, {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, {"%csv_header", "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," From 37dbe80f9b190b263f6eb437e93b9c8fa4df2849 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 12 Feb 2018 21:28:23 -0700 Subject: [PATCH 004/330] Attempt to fix travis timeouts during apt-get. (#528) * Attempt to fix travis timeouts during apt-get. During some builds, travis fails to update the apt-get indexes. This causes the build to fail in different ways. This patch attempts to avoid this issue by manually calling apt-get update. I'm not sure if it'll work, but it's worth a try. * Fix missing semicolons in command --- .travis.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58be73f06c..a52683f15b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,13 +31,8 @@ matrix: - g++-multilib env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON - compiler: gcc - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-6 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold" - compiler: clang @@ -144,7 +139,16 @@ before_script: fi - mkdir -p build && cd build +before_install: + - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then + sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test"; + sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"; + fi + install: + - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then + sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install g++-6; + fi - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then PATH=~/.local/bin:${PATH}; pip install --user --upgrade pip; From 3924ee7b8a6c6427083662c861c0f51be2a38bd9 Mon Sep 17 00:00:00 2001 From: Samuel Panzer Date: Tue, 13 Feb 2018 15:54:46 -0500 Subject: [PATCH 005/330] Fixups following addition of KeepRunningBatch (296ec5693) (#526) * Support State::KeepRunningBatch(). State::KeepRunning() can take large amounts of time relative to quick operations (on the order of 1ns, depending on hardware). For such sensitive operations, it is recommended to run batches of repeated operations. This commit simplifies handling of total_iterations_. Rather than predecrementing such that total_iterations_ == 1 signals that KeepRunning() should exit, total_iterations_ == 0 now signals the intention for the benchmark to exit. * Create better fast path in State::KeepRunningBatch() * Replace int parameter with size_t to fix signed mismatch warnings * Ensure benchmark State has been started even on error. * Simplify KeepRunningBatch() * Implement KeepRunning() in terms of KeepRunningBatch(). * Improve codegen by helping the compiler undestand dead code. * Dummy commit for build bots' benefit. --- include/benchmark/benchmark.h | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index a7ac9649e6..1825054bd5 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -433,6 +433,7 @@ class State { bool KeepRunning(); // Returns true iff the benchmark should run n more iterations. + // REQUIRES: 'n' > 0. // NOTE: A benchmark must not return from the test until KeepRunningBatch() // has returned false. // NOTE: KeepRunningBatch() may overshoot by up to 'n' iterations. @@ -609,6 +610,9 @@ class State { private: void StartKeepRunning(); + // Implementation of KeepRunning() and KeepRunningBatch(). + // is_batch must be true unless n is 1. + bool KeepRunningInternal(size_t n, bool is_batch); void FinishKeepRunning(); internal::ThreadTimer* timer_; internal::ThreadManager* manager_; @@ -617,28 +621,21 @@ class State { inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { - // total_iterations_ is set to 0 by the constructor, and always set to a - // nonzero value by StartKepRunning(). - if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ != 0, true)) { - --total_iterations_; - return true; - } - if (!started_) { - StartKeepRunning(); - if (!error_occurred_) { - // max_iterations > 0. The first iteration is always valid. - --total_iterations_; - return true; - } - } - FinishKeepRunning(); - return false; + return KeepRunningInternal(1, /*is_batch=*/ false); } inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(size_t n) { + return KeepRunningInternal(n, /*is_batch=*/ true); +} + +inline BENCHMARK_ALWAYS_INLINE +bool State::KeepRunningInternal(size_t n, bool is_batch) { // total_iterations_ is set to 0 by the constructor, and always set to a // nonzero value by StartKepRunning(). + assert(n > 0); + // n must be 1 unless is_batch is true. + assert(is_batch || n == 1); if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) { total_iterations_ -= n; return true; @@ -650,7 +647,8 @@ bool State::KeepRunningBatch(size_t n) { return true; } } - if (total_iterations_ != 0) { + // For non-batch runs, total_iterations_ must be 0 by now. + if (is_batch && total_iterations_ != 0) { batch_leftover_ = n - total_iterations_; total_iterations_ = 0; return true; From 207b9c7aeccd70a1c11a27504e1627ef9f8b5938 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 14 Feb 2018 13:44:41 -0700 Subject: [PATCH 006/330] Improve State packing: put important members on first cache line. (#527) * Improve State packing: put important members on first cache line. This patch does a few different things to ensure commonly accessed data is on the first cache line of the `State` object. First, it moves the `error_occurred_` member to reside after the `started_` and `finished_` bools, since there was internal padding there that was unused. Second, it moves `batch_leftover_` and `max_iterations` further up in the struct declaration. These variables are used in the calculation of `iterations()` which users might call within the loop. Therefore it's more important they exist on the first cache line. Finally, this patch turns the bool members into bitfields. Although this shouldn't have much of an effect currently, because padding is still needed between the last bool and the first size_t, it should help in future changes that require more "bool like" members. * Remove bitfield change for now * Move bools (and their padding) to end of "first cache line" vars. I think it makes the most sense to move the padding required following the group of bools to the end of the variables we want on the first cache line. This also means that the `total_iterations_` variable, which is the most accessed, has the same address as the State object. * Fix static assertion after moving bools --- CMakeLists.txt | 7 +++++++ cmake/AddCXXCompilerFlag.cmake | 10 ++++++++++ include/benchmark/benchmark.h | 30 ++++++++++++++++++------------ src/benchmark.cc | 15 ++++++++++----- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa0826760c..e25f7fae27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,6 +120,13 @@ else() cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) endif() + # GCC doesn't report invalid -Wno-warning flags, so check the positive + # version, and if it exists add the negative. + check_cxx_warning_flag(-Winvalid-offsetof) + if (HAVE_CXX_FLAG_WINVALID_OFFSETOF) + add_cxx_compiler_flag(-Wno-invalid-offsetof) + endif() + # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. # However g++ doesn't do this in Cygwin so we have to define it ourselfs diff --git a/cmake/AddCXXCompilerFlag.cmake b/cmake/AddCXXCompilerFlag.cmake index 17d5f3dcc3..d0d2099814 100644 --- a/cmake/AddCXXCompilerFlag.cmake +++ b/cmake/AddCXXCompilerFlag.cmake @@ -62,3 +62,13 @@ function(add_required_cxx_compiler_flag FLAG) message(FATAL_ERROR "Required flag '${FLAG}' is not supported by the compiler") endif() endfunction() + +function(check_cxx_warning_flag FLAG) + mangle_compiler_flag("${FLAG}" MANGLED_FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + # Add -Werror to ensure the compiler generates an error if the warning flag + # doesn't exist. + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") +endfunction() diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 1825054bd5..5fed767d86 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -573,13 +573,26 @@ class State { return (max_iterations - total_iterations_ + batch_leftover_); } - private: - bool started_; - bool finished_; +private: // items we expect on the first cache line (ie 64 bytes of the struct) + // When total_iterations_ is 0, KeepRunning() and friends will return false. - size_t total_iterations_; // May be larger than max_iterations. + size_t total_iterations_; + + // When using KeepRunningBatch(), batch_leftover_ holds the number of + // iterations beyond max_iters that were run. Used to track + // completed_iterations_ accurately. + size_t batch_leftover_; + +public: + const size_t max_iterations; +private: + bool started_; + bool finished_; + bool error_occurred_; + +private: // items we don't need on the first cache line std::vector range_; size_t bytes_processed_; @@ -587,13 +600,6 @@ class State { int complexity_n_; - bool error_occurred_; - - // When using KeepRunningBatch(), batch_leftover_ holds the number of - // iterations beyond max_iters that were run. Used to track - // completed_iterations_ accurately. - size_t batch_leftover_; - public: // Container for user-defined counters. UserCounters counters; @@ -601,7 +607,7 @@ class State { const int thread_index; // Number of threads concurrently executing the benchmark. const int threads; - const size_t max_iterations; + // TODO(EricWF) make me private State(size_t max_iters, const std::vector& ranges, int thread_i, diff --git a/src/benchmark.cc b/src/benchmark.cc index 8879204aca..e0fb6f920a 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -397,23 +397,28 @@ std::vector RunBenchmark( State::State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager) - : started_(false), + : total_iterations_(0), + batch_leftover_(0), + max_iterations(max_iters), + started_(false), finished_(false), - total_iterations_(0), + error_occurred_(false), range_(ranges), bytes_processed_(0), items_processed_(0), complexity_n_(0), - error_occurred_(false), - batch_leftover_(0), counters(), thread_index(thread_i), threads(n_threads), - max_iterations(max_iters), timer_(timer), manager_(manager) { CHECK(max_iterations != 0) << "At least one iteration must be run"; CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; + + // Offset tests to ensure commonly accessed data is on the first cache line. + const int cache_line_size = 64; + static_assert(offsetof(State, error_occurred_) <= + (cache_line_size - sizeof(error_occurred_)), ""); } void State::PauseTiming() { From 6ecf8a8e80b92a7292558ae32466b4df331c851d Mon Sep 17 00:00:00 2001 From: Ian McKellar Date: Wed, 14 Feb 2018 13:17:12 -0800 Subject: [PATCH 007/330] Don't include on Fuchsia. (#531) * Don't include on Fuchsia. It doesn't support POSIX resource measurement and timing APIs. Change-Id: Ifab4bac4296575f042c699db1ce5a4f7c2d82893 * Add BENCHMARK_OS_FUCHSIA for Fuchsia Change-Id: Ic536f9625e413270285fbfd08471dcb6753ddad1 --- src/benchmark.cc | 2 ++ src/benchmark_register.cc | 2 ++ src/internal_macros.h | 2 ++ src/sysinfo.cc | 2 ++ src/timers.cc | 4 +++- 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index e0fb6f920a..ee9ec5bf89 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -17,7 +17,9 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS +#ifndef BENCHMARK_OS_FUCHSIA #include +#endif #include #include #endif diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index d5746a3632..b6db262c99 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -17,7 +17,9 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS +#ifndef BENCHMARK_OS_FUCHSIA #include +#endif #include #include #endif diff --git a/src/internal_macros.h b/src/internal_macros.h index c34f5716e6..faa753e3fe 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -58,6 +58,8 @@ #define BENCHMARK_OS_EMSCRIPTEN 1 #elif defined(__rtems__) #define BENCHMARK_OS_RTEMS 1 +#elif defined(__Fuchsia__) +#define BENCHMARK_OS_FUCHSIA 1 #endif #if !__has_feature(cxx_exceptions) && !defined(__cpp_exceptions) \ diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 90be166967..2b4e1dd157 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -20,7 +20,9 @@ #include #else #include +#ifndef BENCHMARK_OS_FUCHSIA #include +#endif #include #include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include diff --git a/src/timers.cc b/src/timers.cc index 817272d00b..09680c780c 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -21,7 +21,9 @@ #include #else #include +#ifndef BENCHMARK_OS_FUCHSIA #include +#endif #include #include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include @@ -74,7 +76,7 @@ double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { static_cast(user.QuadPart)) * 1e-7; } -#else +#elif !defined(BENCHMARK_OS_FUCHSIA) double MakeTime(struct rusage const& ru) { return (static_cast(ru.ru_utime.tv_sec) + static_cast(ru.ru_utime.tv_usec) * 1e-6 + From 858688b845b86dc43a7e23f8ec0f94a8e63bfe20 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 21 Feb 2018 00:54:19 -0700 Subject: [PATCH 008/330] Ensure std::iterator_traits instantiates. Due to ADL lookup performed on the begin and end functions of `for (auto _ : State)`, std::iterator_traits may get incidentally instantiated. This patch ensures the library can tolerate that. --- include/benchmark/benchmark.h | 1 + test/basic_test.cc | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 5fed767d86..1eeaef6413 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -669,6 +669,7 @@ struct State::StateIterator { typedef Value value_type; typedef Value reference; typedef Value pointer; + typedef std::ptrdiff_t difference_type; private: friend class State; diff --git a/test/basic_test.cc b/test/basic_test.cc index 12579c0945..100f68985c 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -126,4 +126,10 @@ void BM_RangedFor(benchmark::State& state) { } BENCHMARK(BM_RangedFor); +// Ensure that StateIterator provides all the necessary typedefs required to +// instantiate std::iterator_traits. +static_assert(std::is_same< + typename std::iterator_traits::value_type, + typename benchmark::State::StateIterator::value_type>::value, ""); + BENCHMARK_MAIN(); From 19048b7b65875e08c1e882e644a7a5f9bcfd3f82 Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Wed, 21 Feb 2018 16:41:52 +0000 Subject: [PATCH 009/330] Fix typo in README.md (#535) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6bd81e701c..7c77fc2068 100644 --- a/README.md +++ b/README.md @@ -821,7 +821,7 @@ BM_SetInsert/1024/10 33157 33648 21431 1.13369M The JSON format outputs human readable json split into two top level attributes. The `context` attribute contains information about the run in general, including information about the CPU and the date. -The `benchmarks` attribute contains a list of ever benchmark run. Example json +The `benchmarks` attribute contains a list of every benchmark run. Example json output looks like: ```json { From e9a49be7f1fd6e3717687e42ad318957e8cd6c8e Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Wed, 21 Feb 2018 16:42:16 +0000 Subject: [PATCH 010/330] Update note about linking with pthreads (#536) --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7c77fc2068..0fb90240a5 100644 --- a/README.md +++ b/README.md @@ -893,8 +893,11 @@ If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake cach If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. ## Linking against the library -When using gcc, it is necessary to link against pthread to avoid runtime exceptions. -This is due to how gcc implements std::thread. + +When the library is built using GCC it is necessary to link with `-pthread`, +due to how GCC implements `std::thread`. + +For GCC 4.x failing to link to pthreads will lead to runtime exceptions, not linker errors. See [issue #67](https://github.com/google/benchmark/issues/67) for more details. ## Compiler Support From 56f52ee228783547f544d9ac4a533574b9010e3f Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 21 Feb 2018 09:43:57 -0700 Subject: [PATCH 011/330] Print the executable name as part of the context. (#534) * Print the executable name as part of the context. A common use case of the library is to run two different versions of a benchmark to compare them. In my experience this often means compiling a benchmark twice, renaming one of the executables, and then running the executables back-to-back. In this case the name of the executable is important contextually information. Unfortunately the benchmark does not report this information. This patch adds the executable name to the context reported by the benchmark. * attempt to fix tests on Windows * attempt to fix tests on Windows --- include/benchmark/benchmark.h | 2 +- src/benchmark.cc | 1 + src/json_reporter.cc | 4 ++++ src/reporter.cc | 6 ++++++ test/reporter_output_test.cc | 2 ++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 1eeaef6413..002331c4ad 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1247,7 +1247,7 @@ class BenchmarkReporter { CPUInfo const& cpu_info; // The number of chars in the longest benchmark name. size_t name_field_width; - + static const char *executable_name; Context(); }; diff --git a/src/benchmark.cc b/src/benchmark.cc index ee9ec5bf89..2bfd19a778 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -661,6 +661,7 @@ void PrintUsageAndExit() { void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; + BenchmarkReporter::Context::executable_name = argv[0]; for (int i = 1; i < *argc; ++i) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || diff --git a/src/json_reporter.cc b/src/json_reporter.cc index b5ae302ad4..254ec3eeba 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -77,6 +77,10 @@ bool JSONReporter::ReportContext(const Context& context) { std::string walltime_value = LocalDateTimeString(); out << indent << FormatKV("date", walltime_value) << ",\n"; + if (Context::executable_name) { + out << indent << FormatKV("executable", Context::executable_name) << ",\n"; + } + CPUInfo const& info = context.cpu_info; out << indent << FormatKV("num_cpus", static_cast(info.num_cpus)) << ",\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 5d2fa05a2b..4b40aaec8b 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -37,6 +37,9 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << LocalDateTimeString() << "\n"; + if (context.executable_name) + Out << "Running " << context.executable_name << "\n"; + const CPUInfo &info = context.cpu_info; Out << "Run on (" << info.num_cpus << " X " << (info.cycles_per_second / 1000000.0) << " MHz CPU " @@ -64,6 +67,9 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, #endif } +// No initializer because it's already initialized to NULL. +const char* BenchmarkReporter::Context::executable_name; + BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()) {} double BenchmarkReporter::Run::GetAdjustedRealTime() const { diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 1620b31396..23eb1baf63 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -17,11 +17,13 @@ static int AddContextCases() { AddCases(TC_ConsoleErr, { {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, + {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s\\)", MR_Next}, }); AddCases(TC_JSONOut, {{"^\\{", MR_Default}, {"\"context\":", MR_Next}, {"\"date\": \"", MR_Next}, + {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next}, {"\"num_cpus\": %int,$", MR_Next}, {"\"mhz_per_cpu\": %float,$", MR_Next}, {"\"cpu_scaling_enabled\": ", MR_Next}, From ff2c255af5bb2fc2e5cd3b3685f0c6283117ce73 Mon Sep 17 00:00:00 2001 From: Robert Guo Date: Fri, 2 Mar 2018 06:22:03 -0500 Subject: [PATCH 012/330] Use STCK to get the CPU clock on s390x (#540) --- AUTHORS | 1 + CONTRIBUTORS | 1 + src/cycleclock.h | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/AUTHORS b/AUTHORS index 4e4c4ed475..e574b23bbb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -31,6 +31,7 @@ Kishan Kumar Lei Xu Matt Clarkson Maxim Vafin +MongoDB Inc. Nick Hutchinson Oleksandr Sochka Paul Redmond diff --git a/CONTRIBUTORS b/CONTRIBUTORS index c59134b9bd..b942722e3b 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -53,6 +53,7 @@ Pierre Phaneuf Radoslav Yovchev Raul Marin Ray Glover +Robert Guo Roman Lebedev Shuo Chen Tobias Ulvgård diff --git a/src/cycleclock.h b/src/cycleclock.h index 4251fe4c32..3b376ac57d 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -159,6 +159,11 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__s390__) // Covers both s390 and s390x. + // Return the CPU clock. + uint64_t tsc; + asm("stck %0" : "=Q" (tsc) : : "cc"); + return tsc; #else // The soft failover to a generic implementation is automatic only for ARM. // For other platforms the developer is expected to make an attempt to create From 47df49e5731fd795043bd40c0285ba44ab3e2dbb Mon Sep 17 00:00:00 2001 From: alekseyshl Date: Fri, 2 Mar 2018 03:53:58 -0800 Subject: [PATCH 013/330] Add Solaris support (#539) * Add Solaris support Define BENCHMARK_OS_SOLARIS for Solaris. Platform specific implementations added: * number of CPUs detection * CPU cycles per second detection * Thread CPU usage * Process CPU usage * Remove the special case for per process CPU time for Solaris, it's the same as the default. --- README.md | 3 +++ src/CMakeLists.txt | 5 +++++ src/internal_macros.h | 2 ++ src/sysinfo.cc | 41 +++++++++++++++++++++++++++++++++++++++++ src/timers.cc | 4 ++++ 5 files changed, 55 insertions(+) diff --git a/README.md b/README.md index 0fb90240a5..167bf7a266 100644 --- a/README.md +++ b/README.md @@ -936,3 +936,6 @@ sudo cpupower frequency-set --governor powersave * Users must manually link `shlwapi.lib`. Failure to do so may result in unresolved symbols. +### Solaris + +* Users must explicitly link with kstat library (-lkstat compilation flag). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e22620a729..836549e3ca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_link_libraries(benchmark Shlwapi) endif() +# We need extra libraries on Solaris +if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + target_link_libraries(benchmark kstat) +endif() + set(include_install_dir "include") set(lib_install_dir "lib/") set(bin_install_dir "bin/") diff --git a/src/internal_macros.h b/src/internal_macros.h index faa753e3fe..f299722c65 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -60,6 +60,8 @@ #define BENCHMARK_OS_RTEMS 1 #elif defined(__Fuchsia__) #define BENCHMARK_OS_FUCHSIA 1 +#elif defined (__SVR4) && defined (__sun) +#define BENCHMARK_OS_SOLARIS 1 #endif #if !__has_feature(cxx_exceptions) && !defined(__cpp_exceptions) \ diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 2b4e1dd157..fdcc95bb86 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -32,6 +32,9 @@ #include #endif #endif +#if defined(BENCHMARK_OS_SOLARIS) +#include +#endif #include #include @@ -356,6 +359,15 @@ int GetNumCPUs() { return sysinfo.dwNumberOfProcessors; // number of logical // processors in the current // group +#elif defined(BENCHMARK_OS_SOLARIS) + // Returns -1 in case of a failure. + int NumCPU = sysconf(_SC_NPROCESSORS_ONLN); + if (NumCPU < 0) { + fprintf(stderr, + "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n", + strerror(errno)); + } + return NumCPU; #else int NumCPUs = 0; int MaxID = -1; @@ -495,6 +507,35 @@ double GetCPUCyclesPerSecond() { "~MHz", nullptr, &data, &data_size))) return static_cast((int64_t)data * (int64_t)(1000 * 1000)); // was mhz +#elif defined (BENCHMARK_OS_SOLARIS) + kstat_ctl_t *kc = kstat_open(); + if (!kc) { + std::cerr << "failed to open /dev/kstat\n"; + return -1; + } + kstat_t *ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); + if (!ksp) { + std::cerr << "failed to lookup in /dev/kstat\n"; + return -1; + } + if (kstat_read(kc, ksp, NULL) < 0) { + std::cerr << "failed to read from /dev/kstat\n"; + return -1; + } + kstat_named_t *knp = + (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); + if (!knp) { + std::cerr << "failed to lookup data in /dev/kstat\n"; + return -1; + } + if (knp->data_type != KSTAT_DATA_UINT64) { + std::cerr << "current_clock_Hz is of unexpected data type: " + << knp->data_type << "\n"; + return -1; + } + double clock_hz = knp->value.ui64; + kstat_close(kc); + return clock_hz; #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. const int estimate_time_ms = 1000; diff --git a/src/timers.cc b/src/timers.cc index 09680c780c..893cd3827a 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -164,6 +164,10 @@ double ThreadCPUUsage() { // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See // https://github.com/RTEMS/rtems/blob/master/cpukit/posix/src/clockgettime.c return ProcessCPUUsage(); +#elif defined(BENCHMARK_OS_SOLARIS) + struct rusage ru; + if (getrusage(RUSAGE_LWP, &ru) == 0) return MakeTime(ru); + DiagnoseAndExit("getrusage(RUSAGE_LWP, ...) failed"); #elif defined(CLOCK_THREAD_CPUTIME_ID) struct timespec ts; if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) return MakeTime(ts); From 69a52cff4fdc6a12b639c0d2ec532f6f4ab65c48 Mon Sep 17 00:00:00 2001 From: Wink Saville Date: Tue, 6 Mar 2018 03:44:25 -0800 Subject: [PATCH 014/330] Spelling fixes (#543) Upstream spelling fix changes from Pony, ec47ba8f565726414552f4bbf97d7, by ka7@la-evento.com that effected google/benchmark. --- .ycm_extra_conf.py | 2 +- src/benchmark.cc | 4 ++-- src/complexity.cc | 4 ++-- src/sysinfo.cc | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py index 86194357da..5649ddcc74 100644 --- a/.ycm_extra_conf.py +++ b/.ycm_extra_conf.py @@ -7,7 +7,7 @@ flags = [ '-Wall', '-Werror', -'-pendantic-errors', +'-pedantic-errors', '-std=c++0x', '-fno-strict-aliasing', '-O3', diff --git a/src/benchmark.cc b/src/benchmark.cc index 2bfd19a778..91984f73a0 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -84,7 +84,7 @@ DEFINE_string(benchmark_out_format, "json", "The format to use for file output. Valid values are " "'console', 'json', or 'csv'."); -DEFINE_string(benchmark_out, "", "The file to write additonal output to"); +DEFINE_string(benchmark_out, "", "The file to write additional output to"); DEFINE_string(benchmark_color, "auto", "Whether to use colors in the output. Valid values: " @@ -503,7 +503,7 @@ void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter::Context context; context.name_field_width = name_field_width; - // Keep track of runing times of all instances of current benchmark + // Keep track of running times of all instances of current benchmark std::vector complexity_reports; // We flush streams after invoking reporter methods that write to them. This diff --git a/src/complexity.cc b/src/complexity.cc index 88832698ef..b1aad695a4 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -65,10 +65,10 @@ std::string GetBigOString(BigO complexity) { // Find the coefficient for the high-order term in the running time, by // minimizing the sum of squares of relative error, for the fitting curve -// given by the lambda expresion. +// given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](int n) {return n; };). // For a deeper explanation on the algorithm logic, look the README file at // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit diff --git a/src/sysinfo.cc b/src/sysinfo.cc index fdcc95bb86..24ab6d6bf9 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -455,7 +455,7 @@ double GetCPUCyclesPerSecond() { std::string value; if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only - // accept postive values. Some environments (virtual machines) report zero, + // accept positive values. Some environments (virtual machines) report zero, // which would cause infinite looping in WallTime_Init. if (startsWithKey(ln, "cpu MHz")) { if (!value.empty()) { From f48a28d12a762feb5469dee397adb91d09bc4cd6 Mon Sep 17 00:00:00 2001 From: Wink Saville Date: Tue, 6 Mar 2018 10:15:03 -0800 Subject: [PATCH 015/330] Do not let StrCat be renamed to lstrcatA (#546) On Windows the Shlwapi.h file has a macro: #define StrCat lstrcatA And benchmark/src/string_util.h defines StrCat and it is renamed to lstrcatA if we don't undef the macro in Shlwapi.h. This is an innocuous bug if string_util.h is included after Shlwapi.h, but it is a compile error if string_util.h is included before Shlwapi.h. This fixes issue #545. --- src/sysinfo.cc | 1 + src/timers.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 24ab6d6bf9..59769e242e 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -16,6 +16,7 @@ #ifdef BENCHMARK_OS_WINDOWS #include +#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include #include #else diff --git a/src/timers.cc b/src/timers.cc index 893cd3827a..701e4577aa 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -17,6 +17,7 @@ #ifdef BENCHMARK_OS_WINDOWS #include +#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include #include #else From 61497236ddc0d797a47ef612831fb6ab34dc5c9d Mon Sep 17 00:00:00 2001 From: Wink Saville Date: Wed, 7 Mar 2018 03:20:06 -0800 Subject: [PATCH 016/330] Make string_util naming more consistent (#547) * Rename StringXxx to StrXxx in string_util.h and its users This makes the naming consistent within string_util and moves is the Abseil convention. * Style guide is 2 spaces before end of line "//" comments * Rename StrPrintF/StringPrintF to StrFormat for absl compatibility. --- include/benchmark/benchmark.h | 2 +- src/benchmark_register.cc | 12 ++++++------ src/json_reporter.cc | 6 +++--- src/string_util.cc | 6 +++--- src/string_util.h | 10 +++++----- src/sysinfo.cc | 2 +- src/timers.cc | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 002331c4ad..9fb1f556a7 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -543,7 +543,7 @@ class State { // static void BM_Compress(benchmark::State& state) { // ... // double compress = input_size / output_size; - // state.SetLabel(StringPrintf("compress:%.1f%%", 100.0*compression)); + // state.SetLabel(StrFormat("compress:%.1f%%", 100.0*compression)); // } // Produces output that looks like: // BM_Compress 50 50 14115038 compress:27.3% diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index b6db262c99..59b3e4da70 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -172,20 +172,20 @@ bool BenchmarkFamilies::FindBenchmarks( const auto& arg_name = family->arg_names_[arg_i]; if (!arg_name.empty()) { instance.name += - StringPrintF("%s:", family->arg_names_[arg_i].c_str()); + StrFormat("%s:", family->arg_names_[arg_i].c_str()); } } - instance.name += StringPrintF("%d", arg); + instance.name += StrFormat("%d", arg); ++arg_i; } if (!IsZero(family->min_time_)) - instance.name += StringPrintF("/min_time:%0.3f", family->min_time_); + instance.name += StrFormat("/min_time:%0.3f", family->min_time_); if (family->iterations_ != 0) - instance.name += StringPrintF("/iterations:%d", family->iterations_); + instance.name += StrFormat("/iterations:%d", family->iterations_); if (family->repetitions_ != 0) - instance.name += StringPrintF("/repeats:%d", family->repetitions_); + instance.name += StrFormat("/repeats:%d", family->repetitions_); if (family->use_manual_time_) { instance.name += "/manual_time"; @@ -195,7 +195,7 @@ bool BenchmarkFamilies::FindBenchmarks( // Add the number of threads used to the name if (!family->thread_counts_.empty()) { - instance.name += StringPrintF("/threads:%d", instance.threads); + instance.name += StrFormat("/threads:%d", instance.threads); } if (re.Match(instance.name)) { diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 254ec3eeba..685d6b097d 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -32,15 +32,15 @@ namespace benchmark { namespace { std::string FormatKV(std::string const& key, std::string const& value) { - return StringPrintF("\"%s\": \"%s\"", key.c_str(), value.c_str()); + return StrFormat("\"%s\": \"%s\"", key.c_str(), value.c_str()); } std::string FormatKV(std::string const& key, const char* value) { - return StringPrintF("\"%s\": \"%s\"", key.c_str(), value); + return StrFormat("\"%s\": \"%s\"", key.c_str(), value); } std::string FormatKV(std::string const& key, bool value) { - return StringPrintF("\"%s\": %s", key.c_str(), value ? "true" : "false"); + return StrFormat("\"%s\": %s", key.c_str(), value ? "true" : "false"); } std::string FormatKV(std::string const& key, int64_t value) { diff --git a/src/string_util.cc b/src/string_util.cc index 29edb2a468..ebc3acebd2 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -122,7 +122,7 @@ std::string HumanReadableNumber(double n, double one_k) { return ToBinaryStringFullySpecified(n, 1.1, 1, one_k); } -std::string StringPrintFImp(const char* msg, va_list args) { +std::string StrFormatImp(const char* msg, va_list args) { // we might need a second shot at this, so pre-emptivly make a copy va_list args_cp; va_copy(args_cp, args); @@ -152,10 +152,10 @@ std::string StringPrintFImp(const char* msg, va_list args) { return std::string(buff_ptr.get()); } -std::string StringPrintF(const char* format, ...) { +std::string StrFormat(const char* format, ...) { va_list args; va_start(args, format); - std::string tmp = StringPrintFImp(format, args); + std::string tmp = StrFormatImp(format, args); va_end(args); return tmp; } diff --git a/src/string_util.h b/src/string_util.h index c3d53bfd33..e70e769872 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -12,23 +12,23 @@ void AppendHumanReadable(int n, std::string* str); std::string HumanReadableNumber(double n, double one_k = 1024.0); -std::string StringPrintF(const char* format, ...); +std::string StrFormat(const char* format, ...); -inline std::ostream& StringCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { +inline std::ostream& StrCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { return out; } template -inline std::ostream& StringCatImp(std::ostream& out, First&& f, +inline std::ostream& StrCatImp(std::ostream& out, First&& f, Rest&&... rest) { out << std::forward(f); - return StringCatImp(out, std::forward(rest)...); + return StrCatImp(out, std::forward(rest)...); } template inline std::string StrCat(Args&&... args) { std::ostringstream ss; - StringCatImp(ss, std::forward(args)...); + StrCatImp(ss, std::forward(args)...); return ss.str(); } diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 59769e242e..dab020b535 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -16,7 +16,7 @@ #ifdef BENCHMARK_OS_WINDOWS #include -#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA +#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include #include #else diff --git a/src/timers.cc b/src/timers.cc index 701e4577aa..f98b98424d 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -17,7 +17,7 @@ #ifdef BENCHMARK_OS_WINDOWS #include -#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA +#undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include #include #else From a9beffda0b89a6995372100456a4ad894d29b93b Mon Sep 17 00:00:00 2001 From: jmillikin-stripe Date: Thu, 8 Mar 2018 04:48:46 -0800 Subject: [PATCH 017/330] Add support for building with Bazel. (#533) * Add myself to CONTRIBUTORS under the corp CLA for Stripe, Inc. * Add support for building with Bazel. Limitations compared to existing CMake rules: * Defaults to using C++11 ``, with an override via Bazel flag `--define` of `google_benchmark.have_regex`. The TravisCI config sets the regex implementation to `posix` because it uses ancient compilers. * Debug vs Opt mode can't be set per test. TravisCI runs all the tests in debug mode to satisfy `diagnostics_test`, which depends on `CHECK` being live. * Set Bazel workspace name so other repos can refer to it by stable name. This is recommended by the Bazel style guide to avoid each dependent workspace defining its own name for the dependency. --- .gitignore | 3 +++ .travis.yml | 11 +++++++++++ AUTHORS | 1 + BUILD.bazel | 21 ++++++++++++++++++++ CONTRIBUTORS | 1 + WORKSPACE | 9 +++++++++ bazel/BUILD | 16 +++++++++++++++ bazel/have_regex.bzl | 7 +++++++ test/BUILD | 47 ++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 116 insertions(+) create mode 100644 BUILD.bazel create mode 100644 WORKSPACE create mode 100644 bazel/BUILD create mode 100644 bazel/have_regex.bzl create mode 100644 test/BUILD diff --git a/.gitignore b/.gitignore index 93d5ced634..aca5f93885 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ build.ninja install_manifest.txt rules.ninja +# bazel output symlinks. +bazel-* + # out-of-source build top-level folders. build/ _build/ diff --git a/.travis.yml b/.travis.yml index a52683f15b..137cc9876c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -159,11 +159,22 @@ install: brew update; brew install gcc@7; fi + - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then + sudo apt-get update -qq; + sudo apt-get install -qq unzip; + wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh; + sudo bash bazel-installer.sh; + fi + - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then + curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-darwin-x86_64.sh; + sudo bash bazel-installer.sh; + fi script: - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure + - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... after_success: - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then diff --git a/AUTHORS b/AUTHORS index e574b23bbb..45adb27ee5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -39,6 +39,7 @@ Radoslav Yovchev Roman Lebedev Shuo Chen Steinar H. Gunderson +Stripe, Inc. Yixuan Qiu Yusuke Suzuki Zbigniew Skowron diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000000..883ccc4439 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,21 @@ +licenses(["notice"]) + +load("//bazel:have_regex.bzl", "have_regex_copts") + +cc_library( + name = "benchmark", + srcs = glob([ + "src/*.cc", + "src/*.h", + ]), + hdrs = ["include/benchmark/benchmark.h"], + copts = have_regex_copts(), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) + +cc_library( + name = "benchmark_internal_headers", + hdrs = glob(["src/*.h"]), + visibility = ["//test:__pkg__"], +) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index b942722e3b..2f1999be74 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -38,6 +38,7 @@ Ismael Jimenez Martinez Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes +John Millikin Jussi Knuuttila Kai Wolf Kishan Kumar diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000000..9ba32fd75e --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,9 @@ +workspace(name = "com_github_google_benchmark") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "com_google_googletest", + commit = "3f0cf6b62ad1eb50d8736538363d3580dd640c3e", # HEAD + remote = "https://github.com/google/googletest", +) diff --git a/bazel/BUILD b/bazel/BUILD new file mode 100644 index 0000000000..dcbd4376b7 --- /dev/null +++ b/bazel/BUILD @@ -0,0 +1,16 @@ +package(default_visibility = ["//:__subpackages__"]) + +config_setting( + name = "have_std_regex", + values = {"define": "google_benchmark.have_regex=std"}, +) + +config_setting( + name = "have_posix_regex", + values = {"define": "google_benchmark.have_regex=posix"}, +) + +config_setting( + name = "have_gnu_posix_regex", + values = {"define": "google_benchmark.have_regex=gnu_posix"}, +) diff --git a/bazel/have_regex.bzl b/bazel/have_regex.bzl new file mode 100644 index 0000000000..6ea5ba01a2 --- /dev/null +++ b/bazel/have_regex.bzl @@ -0,0 +1,7 @@ +def have_regex_copts(): + return select({ + "//bazel:have_std_regex": ["-DHAVE_STD_REGEX"], + "//bazel:have_posix_regex": ["-DHAVE_POSIX_REGEX"], + "//bazel:have_gnu_posix_regex": ["-DHAVE_GNU_POSIX_REGEX"], + "//conditions:default": ["-DHAVE_STD_REGEX"], + }) diff --git a/test/BUILD b/test/BUILD new file mode 100644 index 0000000000..9e5d4936d9 --- /dev/null +++ b/test/BUILD @@ -0,0 +1,47 @@ +load("//bazel:have_regex.bzl", "have_regex_copts") + +NEEDS_GTEST_MAIN = [ + "statistics_test.cc", +] + +TEST_COPTS = [ + "-pedantic", + "-pedantic-errors", + "-std=c++11", +] + have_regex_copts() + +TEST_ARGS = ["--benchmark_min_time=0.01"] + +cc_library( + name = "output_test_helper", + testonly = 1, + srcs = ["output_test_helper.cc"], + hdrs = ["output_test.h"], + copts = TEST_COPTS, + deps = [ + "//:benchmark", + "//:benchmark_internal_headers", + ], +) + +[cc_test( + name = test_src[:-len(".cc")], + size = "small", + srcs = [test_src], + args = TEST_ARGS + ({ + "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], + }).get(test_src, []), + copts = TEST_COPTS + ({ + "cxx03_test.cc": ["-std=c++03"], + # Some of the issues with DoNotOptimize only occur when optimization is enabled + "donotoptimize_test.cc": ["-O3"], + }).get(test_src, []), + deps = [ + ":output_test_helper", + "//:benchmark", + "//:benchmark_internal_headers", + "@com_google_googletest//:gtest", + ] + ( + ["@com_google_googletest//:gtest_main"] if (test_src in NEEDS_GTEST_MAIN) else [] + ), +) for test_src in glob(["*_test.cc"])] From 674d0498b80097d8bc720eaa7980805b5332c7ad Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 16 Mar 2018 10:14:38 +0000 Subject: [PATCH 018/330] Move thread classes out to clean up monolithic code (#554) --- src/benchmark.cc | 112 ++----------------------------------------- src/thread_manager.h | 62 ++++++++++++++++++++++++ src/thread_timer.h | 69 ++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 109 deletions(-) create mode 100644 src/thread_manager.h create mode 100644 src/thread_timer.h diff --git a/src/benchmark.cc b/src/benchmark.cc index 91984f73a0..5082716b72 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -29,10 +29,10 @@ #include #include #include -#include #include #include #include +#include #include #include "check.h" @@ -46,7 +46,8 @@ #include "re.h" #include "statistics.h" #include "string_util.h" -#include "timers.h" +#include "thread_manager.h" +#include "thread_timer.h" DEFINE_bool(benchmark_list_tests, false, "Print a list of benchmarks. This option overrides all other " @@ -110,113 +111,6 @@ namespace internal { void UseCharPointer(char const volatile*) {} -class ThreadManager { - public: - ThreadManager(int num_threads) - : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} - - Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { - return benchmark_mutex_; - } - - bool StartStopBarrier() EXCLUDES(end_cond_mutex_) { - return start_stop_barrier_.wait(); - } - - void NotifyThreadComplete() EXCLUDES(end_cond_mutex_) { - start_stop_barrier_.removeThread(); - if (--alive_threads_ == 0) { - MutexLock lock(end_cond_mutex_); - end_condition_.notify_all(); - } - } - - void WaitForAllThreads() EXCLUDES(end_cond_mutex_) { - MutexLock lock(end_cond_mutex_); - end_condition_.wait(lock.native_handle(), - [this]() { return alive_threads_ == 0; }); - } - - public: - struct Result { - double real_time_used = 0; - double cpu_time_used = 0; - double manual_time_used = 0; - int64_t bytes_processed = 0; - int64_t items_processed = 0; - int complexity_n = 0; - std::string report_label_; - std::string error_message_; - bool has_error_ = false; - UserCounters counters; - }; - GUARDED_BY(GetBenchmarkMutex()) Result results; - - private: - mutable Mutex benchmark_mutex_; - std::atomic alive_threads_; - Barrier start_stop_barrier_; - Mutex end_cond_mutex_; - Condition end_condition_; -}; - -// Timer management class -class ThreadTimer { - public: - ThreadTimer() = default; - - // Called by each thread - void StartTimer() { - running_ = true; - start_real_time_ = ChronoClockNow(); - start_cpu_time_ = ThreadCPUUsage(); - } - - // Called by each thread - void StopTimer() { - CHECK(running_); - running_ = false; - real_time_used_ += ChronoClockNow() - start_real_time_; - // Floating point error can result in the subtraction producing a negative - // time. Guard against that. - cpu_time_used_ += std::max(ThreadCPUUsage() - start_cpu_time_, 0); - } - - // Called by each thread - void SetIterationTime(double seconds) { manual_time_used_ += seconds; } - - bool running() const { return running_; } - - // REQUIRES: timer is not running - double real_time_used() { - CHECK(!running_); - return real_time_used_; - } - - // REQUIRES: timer is not running - double cpu_time_used() { - CHECK(!running_); - return cpu_time_used_; - } - - // REQUIRES: timer is not running - double manual_time_used() { - CHECK(!running_); - return manual_time_used_; - } - - private: - bool running_ = false; // Is the timer running - double start_real_time_ = 0; // If running_ - double start_cpu_time_ = 0; // If running_ - - // Accumulated time so far (does not contain current slice if running_) - double real_time_used_ = 0; - double cpu_time_used_ = 0; - // Manually set iteration time. User sets this with SetIterationTime(seconds). - double manual_time_used_ = 0; -}; - namespace { BenchmarkReporter::Run CreateRunReport( diff --git a/src/thread_manager.h b/src/thread_manager.h new file mode 100644 index 0000000000..f9ee267dcc --- /dev/null +++ b/src/thread_manager.h @@ -0,0 +1,62 @@ +#ifndef BENCHMARK_THREAD_MANAGER_H +#define BENCHMARK_THREAD_MANAGER_H + +#include "mutex.h" + +namespace benchmark { +namespace internal { + +class ThreadManager { + public: + ThreadManager(int num_threads) + : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} + + Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { + return benchmark_mutex_; + } + + bool StartStopBarrier() EXCLUDES(end_cond_mutex_) { + return start_stop_barrier_.wait(); + } + + void NotifyThreadComplete() EXCLUDES(end_cond_mutex_) { + start_stop_barrier_.removeThread(); + if (--alive_threads_ == 0) { + MutexLock lock(end_cond_mutex_); + end_condition_.notify_all(); + } + } + + void WaitForAllThreads() EXCLUDES(end_cond_mutex_) { + MutexLock lock(end_cond_mutex_); + end_condition_.wait(lock.native_handle(), + [this]() { return alive_threads_ == 0; }); + } + + public: + struct Result { + double real_time_used = 0; + double cpu_time_used = 0; + double manual_time_used = 0; + int64_t bytes_processed = 0; + int64_t items_processed = 0; + int complexity_n = 0; + std::string report_label_; + std::string error_message_; + bool has_error_ = false; + UserCounters counters; + }; + GUARDED_BY(GetBenchmarkMutex()) Result results; + + private: + mutable Mutex benchmark_mutex_; + std::atomic alive_threads_; + Barrier start_stop_barrier_; + Mutex end_cond_mutex_; + Condition end_condition_; +}; + +} // namespace internal +} // namespace benchmark + +#endif // BENCHMARK_THREAD_MANAGER_H diff --git a/src/thread_timer.h b/src/thread_timer.h new file mode 100644 index 0000000000..eaf108e017 --- /dev/null +++ b/src/thread_timer.h @@ -0,0 +1,69 @@ +#ifndef BENCHMARK_THREAD_TIMER_H +#define BENCHMARK_THREAD_TIMER_H + +#include "check.h" +#include "timers.h" + +namespace benchmark { +namespace internal { + +class ThreadTimer { + public: + ThreadTimer() = default; + + // Called by each thread + void StartTimer() { + running_ = true; + start_real_time_ = ChronoClockNow(); + start_cpu_time_ = ThreadCPUUsage(); + } + + // Called by each thread + void StopTimer() { + CHECK(running_); + running_ = false; + real_time_used_ += ChronoClockNow() - start_real_time_; + // Floating point error can result in the subtraction producing a negative + // time. Guard against that. + cpu_time_used_ += std::max(ThreadCPUUsage() - start_cpu_time_, 0); + } + + // Called by each thread + void SetIterationTime(double seconds) { manual_time_used_ += seconds; } + + bool running() const { return running_; } + + // REQUIRES: timer is not running + double real_time_used() { + CHECK(!running_); + return real_time_used_; + } + + // REQUIRES: timer is not running + double cpu_time_used() { + CHECK(!running_); + return cpu_time_used_; + } + + // REQUIRES: timer is not running + double manual_time_used() { + CHECK(!running_); + return manual_time_used_; + } + + private: + bool running_ = false; // Is the timer running + double start_real_time_ = 0; // If running_ + double start_cpu_time_ = 0; // If running_ + + // Accumulated time so far (does not contain current slice if running_) + double real_time_used_ = 0; + double cpu_time_used_ = 0; + // Manually set iteration time. User sets this with SetIterationTime(seconds). + double manual_time_used_ = 0; +}; + +} // namespace internal +} // namespace benchmark + +#endif // BENCHMARK_THREAD_TIMER_H From 68e228944e2da1b3702c6d64b01f559c8b8b91f1 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 21 Mar 2018 13:27:04 -0600 Subject: [PATCH 019/330] Fix #538 - gtest.h not found when building with older CMake versions. Older CMake versions, in particular 2.8, don't seem to correctly handle interface include directories. This causes failures when building the tests. Additionally, older CMake versions use a different library install directory than expected (i.e. they use lib/). This caused certain tests to fail to link. This patch fixes both those issues. The first by manually adding the correct include directory when building the tests. The second by specifying the library output directory when configuring the GTest build. --- cmake/HandleGTest.cmake | 11 +++++++++-- test/CMakeLists.txt | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 3cf8e18b2a..11528b89fb 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -31,8 +31,14 @@ macro(build_external_gtest) list(APPEND GTEST_FLAGS "-Wno-unused-function") endif() split_list(GTEST_FLAGS) + set(EXCLUDE_FROM_ALL_OPT "") + set(EXCLUDE_FROM_ALL_VALUE "") + if (${CMAKE_VERSION} VERSION_GREATER "3.0.99") + set(EXCLUDE_FROM_ALL_OPT "EXCLUDE_FROM_ALL") + set(EXCLUDE_FROM_ALL_VALUE "ON") + endif() ExternalProject_Add(googletest - EXCLUDE_FROM_ALL ON + ${EXCLUDE_FROM_ALL_OPT} ${EXCLUDE_FROM_ALL_VALUE} GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG master PREFIX "${CMAKE_BINARY_DIR}/googletest" @@ -42,6 +48,7 @@ macro(build_external_gtest) -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER} -DCMAKE_INSTALL_PREFIX:PATH= + -DCMAKE_INSTALL_LIBDIR:PATH=/lib -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS} -Dgtest_force_shared_crt:BOOL=ON ) @@ -69,7 +76,7 @@ macro(build_external_gtest) add_dependencies(gtest googletest) add_dependencies(gtest_main googletest) set(GTEST_BOTH_LIBRARIES gtest gtest_main) - #set(GTEST_INCLUDE_DIRS ${install_dir}/include) + set(GTEST_INCLUDE_DIRS ${install_dir}/include) endmacro(build_external_gtest) if (BENCHMARK_ENABLE_GTEST_TESTS) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index efce3ba524..ad1bd93bbb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -144,8 +144,11 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) if (TARGET googletest) add_dependencies(${name} googletest) endif() + if (GTEST_INCLUDE_DIRS) + target_include_directories(${name} PRIVATE ${GTEST_INCLUDE_DIRS}) + endif() target_link_libraries(${name} benchmark - "${GTEST_BOTH_LIBRARIES}" ${CMAKE_THREAD_LIBS_INIT}) + ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_gtest) macro(add_gtest name) From e668e2a1baaa969a8e5a9220e3949e3202dea9e8 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 21 Mar 2018 13:47:25 -0600 Subject: [PATCH 020/330] Fix #552 - GCC and Clang warn on possibly invalid offsetof usage. This patch disables the -Winvalid-offsetof warning for GCC and Clang when using it to check the cache lines of the State object. Technically this usage of offsetof is undefined behavior until C++17. However, all major compilers support this application as an extension, as demonstrated by the passing static assert (If a compiler encounters UB during evaluation of a constant expression, that UB must be diagnosed). Unfortunately, Clang and GCC also produce a warning about it. This patch temporarily suppresses the warning using #pragma's in the source file (instead of globally suppressing the warning in the build systems). This way the warning is ignored for both CMake and Bazel builds without having to modify either build system. --- CMakeLists.txt | 7 ------- src/benchmark.cc | 13 +++++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e25f7fae27..aa0826760c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,13 +120,6 @@ else() cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) endif() - # GCC doesn't report invalid -Wno-warning flags, so check the positive - # version, and if it exists add the negative. - check_cxx_warning_flag(-Winvalid-offsetof) - if (HAVE_CXX_FLAG_WINVALID_OFFSETOF) - add_cxx_compiler_flag(-Wno-invalid-offsetof) - endif() - # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. # However g++ doesn't do this in Cygwin so we have to define it ourselfs diff --git a/src/benchmark.cc b/src/benchmark.cc index 5082716b72..356ed548a0 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -311,10 +311,23 @@ State::State(size_t max_iters, const std::vector& ranges, int thread_i, CHECK(max_iterations != 0) << "At least one iteration must be run"; CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; + // Note: The use of offsetof below is technically undefined until C++17 + // because State is not a standard layout type. However, all compilers + // currently provide well-defined behavior as an extension (which is + // demonstrated since constexpr evaluation must diagnose all undefined + // behavior). However, GCC and Clang also warn about this use of offsetof, + // which must be suppressed. +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#endif // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; static_assert(offsetof(State, error_occurred_) <= (cache_line_size - sizeof(error_occurred_)), ""); +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif } void State::PauseTiming() { From df60aeb2667e140a6c6ae93e9e1d8eb3d33d72ab Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 23 Mar 2018 11:45:15 +0000 Subject: [PATCH 021/330] Rely on compiler intrinsics to identify regex engine. (#555) Having the copts set on a per-target level can lead to ODR violations in some cases. Avoid this by ensuring the regex engine is picked through compiler intrinsics in the header directly. --- BUILD.bazel | 3 --- bazel/BUILD | 16 ---------------- bazel/have_regex.bzl | 7 ------- src/internal_macros.h | 1 + src/re.h | 24 ++++++++++++++++++------ test/BUILD | 4 +--- 6 files changed, 20 insertions(+), 35 deletions(-) delete mode 100644 bazel/BUILD delete mode 100644 bazel/have_regex.bzl diff --git a/BUILD.bazel b/BUILD.bazel index 883ccc4439..35605ccb91 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,7 +1,5 @@ licenses(["notice"]) -load("//bazel:have_regex.bzl", "have_regex_copts") - cc_library( name = "benchmark", srcs = glob([ @@ -9,7 +7,6 @@ cc_library( "src/*.h", ]), hdrs = ["include/benchmark/benchmark.h"], - copts = have_regex_copts(), strip_include_prefix = "include", visibility = ["//visibility:public"], ) diff --git a/bazel/BUILD b/bazel/BUILD deleted file mode 100644 index dcbd4376b7..0000000000 --- a/bazel/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -package(default_visibility = ["//:__subpackages__"]) - -config_setting( - name = "have_std_regex", - values = {"define": "google_benchmark.have_regex=std"}, -) - -config_setting( - name = "have_posix_regex", - values = {"define": "google_benchmark.have_regex=posix"}, -) - -config_setting( - name = "have_gnu_posix_regex", - values = {"define": "google_benchmark.have_regex=gnu_posix"}, -) diff --git a/bazel/have_regex.bzl b/bazel/have_regex.bzl deleted file mode 100644 index 6ea5ba01a2..0000000000 --- a/bazel/have_regex.bzl +++ /dev/null @@ -1,7 +0,0 @@ -def have_regex_copts(): - return select({ - "//bazel:have_std_regex": ["-DHAVE_STD_REGEX"], - "//bazel:have_posix_regex": ["-DHAVE_POSIX_REGEX"], - "//bazel:have_gnu_posix_regex": ["-DHAVE_GNU_POSIX_REGEX"], - "//conditions:default": ["-DHAVE_STD_REGEX"], - }) diff --git a/src/internal_macros.h b/src/internal_macros.h index f299722c65..248ef26259 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -39,6 +39,7 @@ #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 #elif defined(__APPLE__) + #define BENCHMARK_OS_APPLE 1 #include "TargetConditionals.h" #if defined(TARGET_OS_MAC) #define BENCHMARK_OS_MACOSX 1 diff --git a/src/re.h b/src/re.h index 01e9736505..924d2f0ba7 100644 --- a/src/re.h +++ b/src/re.h @@ -17,19 +17,31 @@ #include "internal_macros.h" +#if !defined(HAVE_STD_REGEX) && \ + !defined(HAVE_GNU_POSIX_REGEX) && \ + !defined(HAVE_POSIX_REGEX) + // No explicit regex selection; detect based on builtin hints. + #if defined(BENCHMARK_OS_LINUX) || defined(BENCHMARK_OS_APPLE) + #define HAVE_POSIX_REGEX 1 + #elif __cplusplus >= 199711L + #define HAVE_STD_REGEX 1 + #endif +#endif + // Prefer C regex libraries when compiling w/o exceptions so that we can // correctly report errors. -#if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && defined(HAVE_STD_REGEX) && \ +#if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \ + defined(BENCHMARK_HAVE_STD_REGEX) && \ (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) -#undef HAVE_STD_REGEX + #undef HAVE_STD_REGEX #endif #if defined(HAVE_STD_REGEX) -#include + #include #elif defined(HAVE_GNU_POSIX_REGEX) -#include + #include #elif defined(HAVE_POSIX_REGEX) -#include + #include #else #error No regular expression backend was found! #endif @@ -64,7 +76,7 @@ class Regex { #elif defined(HAVE_POSIX_REGEX) || defined(HAVE_GNU_POSIX_REGEX) regex_t re_; #else -#error No regular expression backend implementation available + #error No regular expression backend implementation available #endif }; diff --git a/test/BUILD b/test/BUILD index 9e5d4936d9..c2682210f9 100644 --- a/test/BUILD +++ b/test/BUILD @@ -1,5 +1,3 @@ -load("//bazel:have_regex.bzl", "have_regex_copts") - NEEDS_GTEST_MAIN = [ "statistics_test.cc", ] @@ -8,7 +6,7 @@ TEST_COPTS = [ "-pedantic", "-pedantic-errors", "-std=c++11", -] + have_regex_copts() +] TEST_ARGS = ["--benchmark_min_time=0.01"] From 7b03df7ff76844a39359e9233f31ca8cdb073313 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 23 Mar 2018 16:10:47 -0600 Subject: [PATCH 022/330] Add tests to verify assembler output -- Fix DoNotOptimize. (#530) * Add tests to verify assembler output -- Fix DoNotOptimize. For things like `DoNotOptimize`, `ClobberMemory`, and even `KeepRunning()`, it is important exactly what assembly they generate. However, we currently have no way to test this. Instead it must be manually validated every time a change occurs -- including a change in compiler version. This patch attempts to introduce a way to test the assembled output automatically. It's mirrors how LLVM verifies compiler output, and it uses LLVM FileCheck to run the tests in a similar way. The tests function by generating the assembly for a test in CMake, and then using FileCheck to verify the // CHECK lines in the source file are found in the generated assembly. Currently, the tests only run on 64-bit x86 systems under GCC and Clang, and when FileCheck is found on the system. Additionally, this patch tries to improve the code gen from DoNotOptimize. This should probably be a separate change, but I needed something to test. * Disable assembly tests on Bazel for now * Link FIXME to github issue * Fix Tests on OS X * fix strip_asm.py to work on both Linux and OS X like targets --- .gitignore | 1 + .travis.yml | 24 +++- CMakeLists.txt | 38 +++++++ README.md | 2 + docs/AssemblyTests.md | 147 ++++++++++++++++++++++++ include/benchmark/benchmark.h | 15 ++- test/AssemblyTests.cmake | 45 ++++++++ test/BUILD | 5 +- test/CMakeLists.txt | 25 ++++ test/clobber_memory_assembly_test.cc | 64 +++++++++++ test/donotoptimize_assembly_test.cc | 163 +++++++++++++++++++++++++++ test/state_assembly_test.cc | 66 +++++++++++ tools/strip_asm.py | 151 +++++++++++++++++++++++++ 13 files changed, 734 insertions(+), 12 deletions(-) create mode 100644 docs/AssemblyTests.md create mode 100644 test/AssemblyTests.cmake create mode 100644 test/clobber_memory_assembly_test.cc create mode 100644 test/donotoptimize_assembly_test.cc create mode 100644 test/state_assembly_test.cc create mode 100755 tools/strip_asm.py diff --git a/.gitignore b/.gitignore index aca5f93885..050e46987f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.dylib *.cmake !/cmake/*.cmake +!/test/AssemblyTests.cmake *~ *.pyc __pycache__ diff --git a/.travis.yml b/.travis.yml index 137cc9876c..09c058c544 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ matrix: env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug + - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold" - compiler: clang env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Debug @@ -91,6 +92,7 @@ matrix: env: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" + - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN @@ -102,6 +104,7 @@ matrix: env: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins + - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" # Clang w/ libc++ and MSAN - compiler: clang @@ -112,8 +115,8 @@ matrix: env: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread + - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - - os: osx osx_image: xcode8.3 compiler: clang @@ -131,15 +134,20 @@ matrix: - COMPILER=g++-7 C_COMPILER=gcc-7 BUILD_TYPE=Debug before_script: - - if [ -z "$BUILD_32_BITS" ]; then - export BUILD_32_BITS=OFF && echo disabling 32 bit build; - fi - if [ -n "${LIBCXX_BUILD}" ]; then source .travis-libcxx-setup.sh; fi + - if [ -n "${ENABLE_SANITIZER}" ]; then + export EXTRA_OPTIONS="-DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF"; + else + export EXTRA_OPTIONS=""; + fi - mkdir -p build && cd build before_install: + - if [ -z "$BUILD_32_BITS" ]; then + export BUILD_32_BITS=OFF && echo disabling 32 bit build; + fi - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test"; sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"; @@ -147,7 +155,11 @@ before_install: install: - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install g++-6; + sudo -E apt-get -yq --no-install-suggests --no-install-recommends install g++-6; + fi + - if [ "${TRAVIS_OS_NAME}" == "linux" -a "${BUILD_32_BITS}" == "OFF" ]; then + sudo -E apt-get -y --no-install-suggests --no-install-recommends install llvm-3.9-tools; + sudo cp /usr/lib/llvm-3.9/bin/FileCheck /usr/local/bin/; fi - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then PATH=~/.local/bin:${PATH}; @@ -171,7 +183,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... diff --git a/CMakeLists.txt b/CMakeLists.txt index aa0826760c..4c107937ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,10 +27,48 @@ option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree buildi # in cases where it is not possible to build or find a valid version of gtest. option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) +set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF) +function(should_enable_assembly_tests) + if(CMAKE_BUILD_TYPE) + string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) + if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage") + # FIXME: The --coverage flag needs to be removed when building assembly + # tests for this to work. + return() + endif() + endif() + if (MSVC) + return() + elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") + return() + elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + # FIXME: Make these work on 32 bit builds + return() + elseif(BENCHMARK_BUILD_32_BITS) + # FIXME: Make these work on 32 bit builds + return() + endif() + find_program(LLVM_FILECHECK_EXE FileCheck) + if (LLVM_FILECHECK_EXE) + set(LLVM_FILECHECK_EXE "${LLVM_FILECHECK_EXE}" CACHE PATH "llvm filecheck" FORCE) + message(STATUS "LLVM FileCheck Found: ${LLVM_FILECHECK_EXE}") + else() + message(STATUS "Failed to find LLVM FileCheck") + return() + endif() + set(ENABLE_ASSEMBLY_TESTS_DEFAULT ON PARENT_SCOPE) +endfunction() +should_enable_assembly_tests() + +# This option disables the building and running of the assembly verification tests +option(BENCHMARK_ENABLE_ASSEMBLY_TESTS "Enable building and running the assembly tests" + ${ENABLE_ASSEMBLY_TESTS_DEFAULT}) + # Make sure we can import out CMake functions list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + # Read the git tags to determine the project version include(GetGitVersion) get_git_version(GIT_VERSION) diff --git a/README.md b/README.md index 167bf7a266..c8f8c01f0f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ IRC channel: https://freenode.net #googlebenchmark [Additional Tooling Documentation](docs/tools.md) +[Assembly Testing Documentation](docs/AssemblyTests.md) + ## Building diff --git a/docs/AssemblyTests.md b/docs/AssemblyTests.md new file mode 100644 index 0000000000..1fbdc269b5 --- /dev/null +++ b/docs/AssemblyTests.md @@ -0,0 +1,147 @@ +# Assembly Tests + +The Benchmark library provides a number of functions whose primary +purpose in to affect assembly generation, including `DoNotOptimize` +and `ClobberMemory`. In addition there are other functions, +such as `KeepRunning`, for which generating good assembly is paramount. + +For these functions it's important to have tests that verify the +correctness and quality of the implementation. This requires testing +the code generated by the compiler. + +This document describes how the Benchmark library tests compiler output, +as well as how to properly write new tests. + + +## Anatomy of a Test + +Writing a test has two steps: + +* Write the code you want to generate assembly for. +* Add `// CHECK` lines to match against the verified assembly. + +Example: +```c++ + +// CHECK-LABEL: test_add: +extern "C" int test_add() { + extern int ExternInt; + return ExternInt + 1; + + // CHECK: movl ExternInt(%rip), %eax + // CHECK: addl %eax + // CHECK: ret +} + +``` + +#### LLVM Filecheck + +[LLVM's Filecheck](https://llvm.org/docs/CommandGuide/FileCheck.html) +is used to test the generated assembly against the `// CHECK` lines +specified in the tests source file. Please see the documentation +linked above for information on how to write `CHECK` directives. + +#### Tips and Tricks: + +* Tests should match the minimal amount of output required to establish +correctness. `CHECK` directives don't have to match on the exact next line +after the previous match, so tests should omit checks for unimportant +bits of assembly. ([`CHECK-NEXT`](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-next-directive) +can be used to ensure a match occurs exactly after the previous match). + +* The tests are compiled with `-O3 -g0`. So we're only testing the +optimized output. + +* The assembly output is further cleaned up using `tools/strip_asm.py`. +This removes comments, assembler directives, and unused labels before +the test is run. + +* The generated and stripped assembly file for a test is output under +`/test/.s` + +* Filecheck supports using [`CHECK` prefixes](https://llvm.org/docs/CommandGuide/FileCheck.html#cmdoption-check-prefixes) +to specify lines that should only match in certain situations. +The Benchmark tests use `CHECK-CLANG` and `CHECK-GNU` for lines that +are only expected to match Clang or GCC's output respectively. Normal +`CHECK` lines match against all compilers. (Note: `CHECK-NOT` and +`CHECK-LABEL` are NOT prefixes. They are versions of non-prefixed +`CHECK` lines) + +* Use `extern "C"` to disable name mangling for specific functions. This +makes them easier to name in the `CHECK` lines. + + +## Problems Writing Portable Tests + +Writing tests which check the code generated by a compiler are +inherently non-portable. Different compilers and even different compiler +versions may generate entirely different code. The Benchmark tests +must tolerate this. + +LLVM Filecheck provides a number of mechanisms to help write +"more portable" tests; including [matching using regular expressions](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-pattern-matching-syntax), +allowing the creation of [named variables](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-variables) +for later matching, and [checking non-sequential matches](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-dag-directive). + +#### Capturing Variables + +For example, say GCC stores a variable in a register but Clang stores +it in memory. To write a test that tolerates both cases we "capture" +the destination of the store, and then use the captured expression +to write the remainder of the test. + +```c++ +// CHECK-LABEL: test_div_no_op_into_shr: +extern "C" void test_div_no_op_into_shr(int value) { + int divisor = 2; + benchmark::DoNotOptimize(divisor); // hide the value from the optimizer + return value / divisor; + + // CHECK: movl $2, [[DEST:.*]] + // CHECK: idivl [[DEST]] + // CHECK: ret +} +``` + +#### Using Regular Expressions to Match Differing Output + +Often tests require testing assembly lines which may subtly differ +between compilers or compiler versions. A common example of this +is matching stack frame addresses. In this case regular expressions +can be used to match the differing bits of output. For example: + +```c++ +int ExternInt; +struct Point { int x, y, z; }; + +// CHECK-LABEL: test_store_point: +extern "C" void test_store_point() { + Point p{ExternInt, ExternInt, ExternInt}; + benchmark::DoNotOptimize(p); + + // CHECK: movl ExternInt(%rip), %eax + // CHECK: movl %eax, -{{[0-9]+}}(%rsp) + // CHECK: movl %eax, -{{[0-9]+}}(%rsp) + // CHECK: movl %eax, -{{[0-9]+}}(%rsp) + // CHECK: ret +} +``` + +## Current Requirements and Limitations + +The tests require Filecheck to be installed along the `PATH` of the +build machine. Otherwise the tests will be disabled. + +Additionally, as mentioned in the previous section, codegen tests are +inherently non-portable. Currently the tests are limited to: + +* x86_64 targets. +* Compiled with GCC or Clang + +Further work could be done, at least on a limited basis, to extend the +tests to other architectures and compilers (using `CHECK` prefixes). + +Furthermore, the tests fail for builds which specify additional flags +that modify code generation, including `--coverage` or `-fsanitize=`. + diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 9fb1f556a7..04fbbf4713 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -303,15 +303,20 @@ BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); // See: https://youtu.be/nXaxk27zwlk?t=2441 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - // Clang doesn't like the 'X' constraint on `value` and certain GCC versions - // don't like the 'g' constraint. Attempt to placate them both. +inline BENCHMARK_ALWAYS_INLINE +void DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { #if defined(__clang__) - asm volatile("" : : "g"(value) : "memory"); + asm volatile("" : "+r,m"(value) : : "memory"); #else - asm volatile("" : : "i,r,m"(value) : "memory"); + asm volatile("" : "+m,r"(value) : : "memory"); #endif } + // Force the compiler to flush pending writes to global memory. Acts as an // effective read/write barrier inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { diff --git a/test/AssemblyTests.cmake b/test/AssemblyTests.cmake new file mode 100644 index 0000000000..d8f321aa61 --- /dev/null +++ b/test/AssemblyTests.cmake @@ -0,0 +1,45 @@ + + +set(ASM_TEST_FLAGS "") +check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) +if (BENCHMARK_HAS_O3_FLAG) + list(APPEND ASM_TEST_FLAGS -O3) +endif() + +check_cxx_compiler_flag(-g0 BENCHMARK_HAS_G0_FLAG) +if (BENCHMARK_HAS_G0_FLAG) + list(APPEND ASM_TEST_FLAGS -g0) +endif() + +check_cxx_compiler_flag(-fno-stack-protector BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG) +if (BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG) + list(APPEND ASM_TEST_FLAGS -fno-stack-protector) +endif() + +split_list(ASM_TEST_FLAGS) +string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER) + +macro(add_filecheck_test name) + cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV}) + add_library(${name} OBJECT ${name}.cc) + set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}") + set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s") + add_custom_target(copy_${name} ALL + COMMAND ${PROJECT_SOURCE_DIR}/tools/strip_asm.py + $ + ${ASM_OUTPUT_FILE} + BYPRODUCTS ${ASM_OUTPUT_FILE}) + add_dependencies(copy_${name} ${name}) + if (NOT ARG_CHECK_PREFIXES) + set(ARG_CHECK_PREFIXES "CHECK") + endif() + foreach(prefix ${ARG_CHECK_PREFIXES}) + add_test(NAME run_${name}_${prefix} + COMMAND + ${LLVM_FILECHECK_EXE} ${name}.cc + --input-file=${ASM_OUTPUT_FILE} + --check-prefixes=CHECK,CHECK-${ASM_TEST_COMPILER} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endforeach() +endmacro() + diff --git a/test/BUILD b/test/BUILD index c2682210f9..dfca7dba1a 100644 --- a/test/BUILD +++ b/test/BUILD @@ -42,4 +42,7 @@ cc_library( ] + ( ["@com_google_googletest//:gtest_main"] if (test_src in NEEDS_GTEST_MAIN) else [] ), -) for test_src in glob(["*_test.cc"])] +# FIXME: Add support for assembly tests to bazel. +# See Issue #556 +# https://github.com/google/benchmark/issues/556 +) for test_src in glob(["*_test.cc"], exclude = ["*_assembly_test.cc"])] diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ad1bd93bbb..7c6366f789 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,6 +22,12 @@ if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) endforeach() endif() +check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) +set(BENCHMARK_O3_FLAG "") +if (BENCHMARK_HAS_O3_FLAG) + set(BENCHMARK_O3_FLAG "-O3") +endif() + # NOTE: These flags must be added after find_package(Threads REQUIRED) otherwise # they will break the configuration check. if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) @@ -159,6 +165,25 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_test) endif(BENCHMARK_ENABLE_GTEST_TESTS) +############################################################################### +# Assembly Unit Tests +############################################################################### + +if (BENCHMARK_ENABLE_ASSEMBLY_TESTS) + if (NOT LLVM_FILECHECK_EXE) + message(FATAL_ERROR "LLVM FileCheck is required when including this file") + endif() + include(AssemblyTests.cmake) + add_filecheck_test(donotoptimize_assembly_test) + add_filecheck_test(state_assembly_test) + add_filecheck_test(clobber_memory_assembly_test) +endif() + + + +############################################################################### +# Code Coverage Configuration +############################################################################### # Add the coverage command(s) if(CMAKE_BUILD_TYPE) diff --git a/test/clobber_memory_assembly_test.cc b/test/clobber_memory_assembly_test.cc new file mode 100644 index 0000000000..f41911a39c --- /dev/null +++ b/test/clobber_memory_assembly_test.cc @@ -0,0 +1,64 @@ +#include + +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wreturn-type" +#endif + +extern "C" { + +extern int ExternInt; +extern int ExternInt2; +extern int ExternInt3; + +} + +// CHECK-LABEL: test_basic: +extern "C" void test_basic() { + int x; + benchmark::DoNotOptimize(&x); + x = 101; + benchmark::ClobberMemory(); + // CHECK: leaq [[DEST:[^,]+]], %rax + // CHECK: movl $101, [[DEST]] + // CHECK: ret +} + +// CHECK-LABEL: test_redundant_store: +extern "C" void test_redundant_store() { + ExternInt = 3; + benchmark::ClobberMemory(); + ExternInt = 51; + // CHECK-DAG: ExternInt + // CHECK-DAG: movl $3 + // CHECK: movl $51 +} + +// CHECK-LABEL: test_redundant_read: +extern "C" void test_redundant_read() { + int x; + benchmark::DoNotOptimize(&x); + x = ExternInt; + benchmark::ClobberMemory(); + x = ExternInt2; + // CHECK: leaq [[DEST:[^,]+]], %rax + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, [[DEST]] + // CHECK-NOT: ExternInt2 + // CHECK: ret +} + +// CHECK-LABEL: test_redundant_read2: +extern "C" void test_redundant_read2() { + int x; + benchmark::DoNotOptimize(&x); + x = ExternInt; + benchmark::ClobberMemory(); + x = ExternInt2; + benchmark::ClobberMemory(); + // CHECK: leaq [[DEST:[^,]+]], %rax + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, [[DEST]] + // CHECK: ExternInt2(%rip) + // CHECK: movl %eax, [[DEST]] + // CHECK: ret +} diff --git a/test/donotoptimize_assembly_test.cc b/test/donotoptimize_assembly_test.cc new file mode 100644 index 0000000000..d4b0bab70e --- /dev/null +++ b/test/donotoptimize_assembly_test.cc @@ -0,0 +1,163 @@ +#include + +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wreturn-type" +#endif + +extern "C" { + +extern int ExternInt; +extern int ExternInt2; +extern int ExternInt3; + +inline int Add42(int x) { return x + 42; } + +struct NotTriviallyCopyable { + NotTriviallyCopyable(); + explicit NotTriviallyCopyable(int x) : value(x) {} + NotTriviallyCopyable(NotTriviallyCopyable const&); + int value; +}; + +struct Large { + int value; + int data[2]; +}; + +} +// CHECK-LABEL: test_with_rvalue: +extern "C" void test_with_rvalue() { + benchmark::DoNotOptimize(Add42(0)); + // CHECK: movl $42, %eax + // CHECK: ret +} + +// CHECK-LABEL: test_with_large_rvalue: +extern "C" void test_with_large_rvalue() { + benchmark::DoNotOptimize(Large{ExternInt, {ExternInt, ExternInt}}); + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]] + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: ret +} + +// CHECK-LABEL: test_with_non_trivial_rvalue: +extern "C" void test_with_non_trivial_rvalue() { + benchmark::DoNotOptimize(NotTriviallyCopyable(ExternInt)); + // CHECK: mov{{l|q}} ExternInt(%rip) + // CHECK: ret +} + +// CHECK-LABEL: test_with_lvalue: +extern "C" void test_with_lvalue() { + int x = 101; + benchmark::DoNotOptimize(x); + // CHECK-GNU: movl $101, %eax + // CHECK-CLANG: movl $101, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: ret +} + +// CHECK-LABEL: test_with_large_lvalue: +extern "C" void test_with_large_lvalue() { + Large L{ExternInt, {ExternInt, ExternInt}}; + benchmark::DoNotOptimize(L); + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: ret +} + +// CHECK-LABEL: test_with_non_trivial_lvalue: +extern "C" void test_with_non_trivial_lvalue() { + NotTriviallyCopyable NTC(ExternInt); + benchmark::DoNotOptimize(NTC); + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: ret +} + +// CHECK-LABEL: test_with_const_lvalue: +extern "C" void test_with_const_lvalue() { + const int x = 123; + benchmark::DoNotOptimize(x); + // CHECK: movl $123, %eax + // CHECK: ret +} + +// CHECK-LABEL: test_with_large_const_lvalue: +extern "C" void test_with_large_const_lvalue() { + const Large L{ExternInt, {ExternInt, ExternInt}}; + benchmark::DoNotOptimize(L); + // CHECK: ExternInt(%rip) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: movl %eax, -{{[0-9]+}}(%[[REG]]) + // CHECK: ret +} + +// CHECK-LABEL: test_with_non_trivial_const_lvalue: +extern "C" void test_with_non_trivial_const_lvalue() { + const NotTriviallyCopyable Obj(ExternInt); + benchmark::DoNotOptimize(Obj); + // CHECK: mov{{q|l}} ExternInt(%rip) + // CHECK: ret +} + +// CHECK-LABEL: test_div_by_two: +extern "C" int test_div_by_two(int input) { + int divisor = 2; + benchmark::DoNotOptimize(divisor); + return input / divisor; + // CHECK: movl $2, [[DEST:.*]] + // CHECK: idivl [[DEST]] + // CHECK: ret +} + +// CHECK-LABEL: test_inc_integer: +extern "C" int test_inc_integer() { + int x = 0; + for (int i=0; i < 5; ++i) + benchmark::DoNotOptimize(++x); + // CHECK: movl $1, [[DEST:.*]] + // CHECK: {{(addl \$1,|incl)}} [[DEST]] + // CHECK: {{(addl \$1,|incl)}} [[DEST]] + // CHECK: {{(addl \$1,|incl)}} [[DEST]] + // CHECK: {{(addl \$1,|incl)}} [[DEST]] + // CHECK-CLANG: movl [[DEST]], %eax + // CHECK: ret + return x; +} + +// CHECK-LABEL: test_pointer_rvalue +extern "C" void test_pointer_rvalue() { + // CHECK: movl $42, [[DEST:.*]] + // CHECK: leaq [[DEST]], %rax + // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: ret + int x = 42; + benchmark::DoNotOptimize(&x); +} + +// CHECK-LABEL: test_pointer_const_lvalue: +extern "C" void test_pointer_const_lvalue() { + // CHECK: movl $42, [[DEST:.*]] + // CHECK: leaq [[DEST]], %rax + // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z]+]]) + // CHECK: ret + int x = 42; + int * const xp = &x; + benchmark::DoNotOptimize(xp); +} + +// CHECK-LABEL: test_pointer_lvalue: +extern "C" void test_pointer_lvalue() { + // CHECK: movl $42, [[DEST:.*]] + // CHECK: leaq [[DEST]], %rax + // CHECK-CLANG: movq %rax, -{{[0-9]+}}(%[[REG:[a-z+]+]]) + // CHECK: ret + int x = 42; + int *xp = &x; + benchmark::DoNotOptimize(xp); +} diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc new file mode 100644 index 0000000000..e2c5c8648d --- /dev/null +++ b/test/state_assembly_test.cc @@ -0,0 +1,66 @@ +#include + +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wreturn-type" +#endif + +extern "C" { + extern int ExternInt; + benchmark::State& GetState(); + void Fn(); +} + +using benchmark::State; + +// CHECK-LABEL: test_for_auto_loop: +extern "C" int test_for_auto_loop() { + State& S = GetState(); + int x = 42; + // CHECK: [[CALL:call(q)*]] _ZN9benchmark5State16StartKeepRunningEv + // CHECK-NEXT: testq %rbx, %rbx + // CHECK-NEXT: je [[LOOP_END:.*]] + + for (auto _ : S) { + // CHECK: .L[[LOOP_HEAD:[a-zA-Z0-9_]+]]: + // CHECK-GNU-NEXT: subq $1, %rbx + // CHECK-CLANG-NEXT: {{(addq \$1,|incq)}} %rax + // CHECK-NEXT: jne .L[[LOOP_HEAD]] + benchmark::DoNotOptimize(x); + } + // CHECK: [[LOOP_END]]: + // CHECK: [[CALL]] _ZN9benchmark5State17FinishKeepRunningEv + + // CHECK: movl $101, %eax + // CHECK: ret + return 101; +} + +// CHECK-LABEL: test_while_loop: +extern "C" int test_while_loop() { + State& S = GetState(); + int x = 42; + + // CHECK: j{{(e|mp)}} .L[[LOOP_HEADER:[a-zA-Z0-9_]+]] + // CHECK-NEXT: .L[[LOOP_BODY:[a-zA-Z0-9_]+]]: + while (S.KeepRunning()) { + // CHECK-GNU-NEXT: subq $1, %[[IREG:[a-z]+]] + // CHECK-CLANG-NEXT: {{(addq \$-1,|decq)}} %[[IREG:[a-z]+]] + // CHECK: movq %[[IREG]], [[DEST:.*]] + benchmark::DoNotOptimize(x); + } + // CHECK-DAG: movq [[DEST]], %[[IREG]] + // CHECK-DAG: testq %[[IREG]], %[[IREG]] + // CHECK-DAG: jne .L[[LOOP_BODY]] + // CHECK-DAG: .L[[LOOP_HEADER]]: + + // CHECK: cmpb $0 + // CHECK-NEXT: jne .L[[LOOP_END:[a-zA-Z0-9_]+]] + // CHECK: [[CALL:call(q)*]] _ZN9benchmark5State16StartKeepRunningEv + + // CHECK: .L[[LOOP_END]]: + // CHECK: [[CALL]] _ZN9benchmark5State17FinishKeepRunningEv + + // CHECK: movl $101, %eax + // CHECK: ret + return 101; +} diff --git a/tools/strip_asm.py b/tools/strip_asm.py new file mode 100755 index 0000000000..9030550b43 --- /dev/null +++ b/tools/strip_asm.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python + +""" +strip_asm.py - Cleanup ASM output for the specified file +""" + +from argparse import ArgumentParser +import sys +import os +import re + +def find_used_labels(asm): + found = set() + label_re = re.compile("\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") + for l in asm.splitlines(): + m = label_re.match(l) + if m: + found.add('.L%s' % m.group(1)) + return found + + +def normalize_labels(asm): + decls = set() + label_decl = re.compile("^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for l in asm.splitlines(): + m = label_decl.match(l) + if m: + decls.add(m.group(0)) + if len(decls) == 0: + return asm + needs_dot = next(iter(decls))[0] != '.' + if not needs_dot: + return asm + for ld in decls: + asm = re.sub("(^|\s+)" + ld + "(?=:|\s)", '\\1.' + ld, asm) + return asm + + +def transform_labels(asm): + asm = normalize_labels(asm) + used_decls = find_used_labels(asm) + new_asm = '' + label_decl = re.compile("^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for l in asm.splitlines(): + m = label_decl.match(l) + if not m or m.group(0) in used_decls: + new_asm += l + new_asm += '\n' + return new_asm + + +def is_identifier(tk): + if len(tk) == 0: + return False + first = tk[0] + if not first.isalpha() and first != '_': + return False + for i in range(1, len(tk)): + c = tk[i] + if not c.isalnum() and c != '_': + return False + return True + +def process_identifiers(l): + """ + process_identifiers - process all identifiers and modify them to have + consistent names across all platforms; specifically across ELF and MachO. + For example, MachO inserts an additional understore at the beginning of + names. This function removes that. + """ + parts = re.split(r'([a-zA-Z0-9_]+)', l) + new_line = '' + for tk in parts: + if is_identifier(tk): + if tk.startswith('__Z'): + tk = tk[1:] + elif tk.startswith('_') and len(tk) > 1 and \ + tk[1].isalpha() and tk[1] != 'Z': + tk = tk[1:] + new_line += tk + return new_line + + +def process_asm(asm): + """ + Strip the ASM of unwanted directives and lines + """ + new_contents = '' + asm = transform_labels(asm) + + # TODO: Add more things we want to remove + discard_regexes = [ + re.compile("\s+\..*$"), # directive + re.compile("\s*#(NO_APP|APP)$"), #inline ASM + re.compile("\s*#.*$"), # comment line + re.compile("\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)"), #global directive + re.compile("\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)"), + ] + keep_regexes = [ + + ] + fn_label_def = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]*:") + for l in asm.splitlines(): + # Remove Mach-O attribute + l = l.replace('@GOTPCREL', '') + add_line = True + for reg in discard_regexes: + if reg.match(l) is not None: + add_line = False + break + for reg in keep_regexes: + if reg.match(l) is not None: + add_line = True + break + if add_line: + if fn_label_def.match(l) and len(new_contents) != 0: + new_contents += '\n' + l = process_identifiers(l) + new_contents += l + new_contents += '\n' + return new_contents + +def main(): + parser = ArgumentParser( + description='generate a stripped assembly file') + parser.add_argument( + 'input', metavar='input', type=str, nargs=1, + help='An input assembly file') + parser.add_argument( + 'out', metavar='output', type=str, nargs=1, + help='The output file') + args, unknown_args = parser.parse_known_args() + input = args.input[0] + output = args.out[0] + if not os.path.isfile(input): + print(("ERROR: input file '%s' does not exist") % input) + sys.exit(1) + contents = None + with open(input, 'r') as f: + contents = f.read() + new_contents = process_asm(contents) + with open(output, 'w') as f: + f.write(new_contents) + + +if __name__ == '__main__': + main() + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off; +# kate: indent-mode python; remove-trailing-spaces modified; From e7eb54b5f8d29f3d91f5fbda0909c65c2ae70ba9 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Sun, 25 Mar 2018 20:05:31 +0100 Subject: [PATCH 023/330] Fix uninitialized warning (#560) --- test/donotoptimize_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/donotoptimize_test.cc b/test/donotoptimize_test.cc index a705654a26..2ce92d1c72 100644 --- a/test/donotoptimize_test.cc +++ b/test/donotoptimize_test.cc @@ -28,13 +28,13 @@ struct BitRef { int main(int, char*[]) { // this test verifies compilation of DoNotOptimize() for some types - char buffer8[8]; + char buffer8[8] = ""; benchmark::DoNotOptimize(buffer8); - char buffer20[20]; + char buffer20[20] = ""; benchmark::DoNotOptimize(buffer20); - char buffer1024[1024]; + char buffer1024[1024] = ""; benchmark::DoNotOptimize(buffer1024); benchmark::DoNotOptimize(&buffer1024[0]); From 9913418d323e64a0111ca0da81388260c2bbe1e9 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 3 Apr 2018 23:12:47 +0100 Subject: [PATCH 024/330] Allow AddRange to work with int64_t. (#548) * Allow AddRange to work with int64_t. Fixes #516 Also, tweak how we manage per-test build needs, and create a standard _gtest suffix for googletest to differentiate from non-googletest tests. I also ran clang-format on the files that I changed (but not the benchmark include or main src as they have too many clang-format issues). * Add benchmark_gtest to cmake * Set(Items|Bytes)Processed now take int64_t --- include/benchmark/benchmark.h | 50 ++++++++-------- src/benchmark.cc | 2 +- src/benchmark_api_internal.h | 2 +- src/benchmark_register.cc | 57 ++++++------------- src/benchmark_register.h | 33 +++++++++++ test/BUILD | 45 +++++++++------ test/CMakeLists.txt | 3 +- test/benchmark_gtest.cc | 33 +++++++++++ test/benchmark_test.cc | 29 +++++----- test/complexity_test.cc | 8 +-- test/map_test.cc | 12 ++-- test/multiple_ranges_test.cc | 33 +++++++++-- ...statistics_test.cc => statistics_gtest.cc} | 0 13 files changed, 191 insertions(+), 116 deletions(-) create mode 100644 src/benchmark_register.h create mode 100644 test/benchmark_gtest.cc rename test/{statistics_test.cc => statistics_gtest.cc} (100%) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 04fbbf4713..ea9743c685 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -514,10 +514,10 @@ class State { // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE - void SetBytesProcessed(size_t bytes) { bytes_processed_ = bytes; } + void SetBytesProcessed(int64_t bytes) { bytes_processed_ = bytes; } BENCHMARK_ALWAYS_INLINE - size_t bytes_processed() const { return bytes_processed_; } + int64_t bytes_processed() const { return bytes_processed_; } // If this routine is called with complexity_n > 0 and complexity report is // requested for the @@ -525,10 +525,10 @@ class State { // and complexity_n will // represent the length of N. BENCHMARK_ALWAYS_INLINE - void SetComplexityN(int complexity_n) { complexity_n_ = complexity_n; } + void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE - int complexity_length_n() { return complexity_n_; } + int64_t complexity_length_n() { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently @@ -537,10 +537,10 @@ class State { // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE - void SetItemsProcessed(size_t items) { items_processed_ = items; } + void SetItemsProcessed(int64_t items) { items_processed_ = items; } BENCHMARK_ALWAYS_INLINE - size_t items_processed() const { return items_processed_; } + int64_t items_processed() const { return items_processed_; } // If this routine is called, the specified label is printed at the // end of the benchmark report line for the currently executing @@ -562,16 +562,16 @@ class State { // Range arguments for this run. CHECKs if the argument has been set. BENCHMARK_ALWAYS_INLINE - int range(std::size_t pos = 0) const { + int64_t range(std::size_t pos = 0) const { assert(range_.size() > pos); return range_[pos]; } BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") - int range_x() const { return range(0); } + int64_t range_x() const { return range(0); } BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") - int range_y() const { return range(1); } + int64_t range_y() const { return range(1); } BENCHMARK_ALWAYS_INLINE size_t iterations() const { @@ -598,12 +598,12 @@ class State { bool error_occurred_; private: // items we don't need on the first cache line - std::vector range_; + std::vector range_; - size_t bytes_processed_; - size_t items_processed_; + int64_t bytes_processed_; + int64_t items_processed_; - int complexity_n_; + int64_t complexity_n_; public: // Container for user-defined counters. @@ -615,7 +615,7 @@ class State { // TODO(EricWF) make me private - State(size_t max_iters, const std::vector& ranges, int thread_i, + State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager); @@ -736,7 +736,7 @@ class Benchmark { // Run this benchmark once with "x" as the extra argument passed // to the function. // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Arg(int x); + Benchmark* Arg(int64_t x); // Run this benchmark with the given time unit for the generated output report Benchmark* Unit(TimeUnit unit); @@ -744,23 +744,23 @@ class Benchmark { // Run this benchmark once for a number of values picked from the // range [start..limit]. (start and limit are always picked.) // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* Range(int start, int limit); + Benchmark* Range(int64_t start, int64_t limit); // Run this benchmark once for all values in the range [start..limit] with // specific step // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* DenseRange(int start, int limit, int step = 1); + Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1); // Run this benchmark once with "args" as the extra arguments passed // to the function. // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Args(const std::vector& args); + Benchmark* Args(const std::vector& args); // Equivalent to Args({x, y}) // NOTE: This is a legacy C++03 interface provided for compatibility only. // New code should use 'Args'. - Benchmark* ArgPair(int x, int y) { - std::vector args; + Benchmark* ArgPair(int64_t x, int64_t y) { + std::vector args; args.push_back(x); args.push_back(y); return Args(args); @@ -769,7 +769,7 @@ class Benchmark { // Run this benchmark once for a number of values picked from the // ranges [start..limit]. (starts and limits are always picked.) // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... - Benchmark* Ranges(const std::vector >& ranges); + Benchmark* Ranges(const std::vector >& ranges); // Equivalent to ArgNames({name}) Benchmark* ArgName(const std::string& name); @@ -781,8 +781,8 @@ class Benchmark { // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). // NOTE: This is a legacy C++03 interface provided for compatibility only. // New code should use 'Ranges'. - Benchmark* RangePair(int lo1, int hi1, int lo2, int hi2) { - std::vector > ranges; + Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) { + std::vector > ranges; ranges.push_back(std::make_pair(lo1, hi1)); ranges.push_back(std::make_pair(lo2, hi2)); return Ranges(ranges); @@ -889,15 +889,13 @@ class Benchmark { int ArgsCnt() const; - static void AddRange(std::vector* dst, int lo, int hi, int mult); - private: friend class BenchmarkFamilies; std::string name_; ReportMode report_mode_; std::vector arg_names_; // Args for all benchmark runs - std::vector > args_; // Args for all benchmark runs + std::vector > args_; // Args for all benchmark runs TimeUnit time_unit_; int range_multiplier_; double min_time_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 356ed548a0..7b0d1136fc 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -290,7 +290,7 @@ std::vector RunBenchmark( } // namespace } // namespace internal -State::State(size_t max_iters, const std::vector& ranges, int thread_i, +State::State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager) : total_iterations_(0), diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index d481dc5286..dd7a3ffe8c 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -17,7 +17,7 @@ struct Benchmark::Instance { std::string name; Benchmark* benchmark; ReportMode report_mode; - std::vector arg; + std::vector arg; TimeUnit time_unit; int range_multiplier; bool use_real_time; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 59b3e4da70..4fea6d915f 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" -#include "benchmark_api_internal.h" -#include "internal_macros.h" +#include "benchmark_register.h" #ifndef BENCHMARK_OS_WINDOWS #ifndef BENCHMARK_OS_FUCHSIA @@ -36,13 +34,16 @@ #include #include +#include "benchmark/benchmark.h" +#include "benchmark_api_internal.h" #include "check.h" #include "commandlineflags.h" #include "complexity.h" -#include "statistics.h" +#include "internal_macros.h" #include "log.h" #include "mutex.h" #include "re.h" +#include "statistics.h" #include "string_util.h" #include "timers.h" @@ -175,7 +176,7 @@ bool BenchmarkFamilies::FindBenchmarks( StrFormat("%s:", family->arg_names_[arg_i].c_str()); } } - + instance.name += StrFormat("%d", arg); ++arg_i; } @@ -246,30 +247,7 @@ Benchmark::Benchmark(const char* name) Benchmark::~Benchmark() {} -void Benchmark::AddRange(std::vector* dst, int lo, int hi, int mult) { - CHECK_GE(lo, 0); - CHECK_GE(hi, lo); - CHECK_GE(mult, 2); - - // Add "lo" - dst->push_back(lo); - - static const int kint32max = std::numeric_limits::max(); - - // Now space out the benchmarks in multiples of "mult" - for (int32_t i = 1; i < kint32max / mult; i *= mult) { - if (i >= hi) break; - if (i > lo) { - dst->push_back(i); - } - } - // Add "hi" (if different from "lo") - if (hi != lo) { - dst->push_back(hi); - } -} - -Benchmark* Benchmark::Arg(int x) { +Benchmark* Benchmark::Arg(int64_t x) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); args_.push_back({x}); return this; @@ -280,20 +258,21 @@ Benchmark* Benchmark::Unit(TimeUnit unit) { return this; } -Benchmark* Benchmark::Range(int start, int limit) { +Benchmark* Benchmark::Range(int64_t start, int64_t limit) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); - std::vector arglist; + std::vector arglist; AddRange(&arglist, start, limit, range_multiplier_); - for (int i : arglist) { + for (int64_t i : arglist) { args_.push_back({i}); } return this; } -Benchmark* Benchmark::Ranges(const std::vector>& ranges) { +Benchmark* Benchmark::Ranges( + const std::vector>& ranges) { CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); - std::vector> arglists(ranges.size()); + std::vector> arglists(ranges.size()); std::size_t total = 1; for (std::size_t i = 0; i < ranges.size(); i++) { AddRange(&arglists[i], ranges[i].first, ranges[i].second, @@ -304,7 +283,7 @@ Benchmark* Benchmark::Ranges(const std::vector>& ranges) { std::vector ctr(arglists.size(), 0); for (std::size_t i = 0; i < total; i++) { - std::vector tmp; + std::vector tmp; tmp.reserve(arglists.size()); for (std::size_t j = 0; j < arglists.size(); j++) { @@ -336,17 +315,17 @@ Benchmark* Benchmark::ArgNames(const std::vector& names) { return this; } -Benchmark* Benchmark::DenseRange(int start, int limit, int step) { +Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); CHECK_GE(start, 0); CHECK_LE(start, limit); - for (int arg = start; arg <= limit; arg += step) { + for (int64_t arg = start; arg <= limit; arg += step) { args_.push_back({arg}); } return this; } -Benchmark* Benchmark::Args(const std::vector& args) { +Benchmark* Benchmark::Args(const std::vector& args) { CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(args.size())); args_.push_back(args); return this; @@ -363,7 +342,6 @@ Benchmark* Benchmark::RangeMultiplier(int multiplier) { return this; } - Benchmark* Benchmark::MinTime(double t) { CHECK(t > 0.0); CHECK(iterations_ == 0); @@ -371,7 +349,6 @@ Benchmark* Benchmark::MinTime(double t) { return this; } - Benchmark* Benchmark::Iterations(size_t n) { CHECK(n > 0); CHECK(IsZero(min_time_)); diff --git a/src/benchmark_register.h b/src/benchmark_register.h new file mode 100644 index 0000000000..0705e219f2 --- /dev/null +++ b/src/benchmark_register.h @@ -0,0 +1,33 @@ +#ifndef BENCHMARK_REGISTER_H +#define BENCHMARK_REGISTER_H + +#include + +#include "check.h" + +template +void AddRange(std::vector* dst, T lo, T hi, int mult) { + CHECK_GE(lo, 0); + CHECK_GE(hi, lo); + CHECK_GE(mult, 2); + + // Add "lo" + dst->push_back(lo); + + static const T kmax = std::numeric_limits::max(); + + // Now space out the benchmarks in multiples of "mult" + for (T i = 1; i < kmax / mult; i *= mult) { + if (i >= hi) break; + if (i > lo) { + dst->push_back(i); + } + } + + // Add "hi" (if different from "lo") + if (hi != lo) { + dst->push_back(hi); + } +} + +#endif // BENCHMARK_REGISTER_H diff --git a/test/BUILD b/test/BUILD index dfca7dba1a..2b3a391296 100644 --- a/test/BUILD +++ b/test/BUILD @@ -1,15 +1,28 @@ -NEEDS_GTEST_MAIN = [ - "statistics_test.cc", -] - TEST_COPTS = [ "-pedantic", "-pedantic-errors", "-std=c++11", + "-Wall", + "-Wextra", + "-Wshadow", +# "-Wshorten-64-to-32", + "-Wfloat-equal", + "-fstrict-aliasing", ] +PER_SRC_COPTS = ({ + "cxx03_test.cc": ["-std=c++03"], + # Some of the issues with DoNotOptimize only occur when optimization is enabled + "donotoptimize_test.cc": ["-O3"], +}) + + TEST_ARGS = ["--benchmark_min_time=0.01"] +PER_SRC_TEST_ARGS = ({ + "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], +}) + cc_library( name = "output_test_helper", testonly = 1, @@ -22,27 +35,23 @@ cc_library( ], ) -[cc_test( +[ + cc_test( name = test_src[:-len(".cc")], size = "small", srcs = [test_src], - args = TEST_ARGS + ({ - "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], - }).get(test_src, []), - copts = TEST_COPTS + ({ - "cxx03_test.cc": ["-std=c++03"], - # Some of the issues with DoNotOptimize only occur when optimization is enabled - "donotoptimize_test.cc": ["-O3"], - }).get(test_src, []), + args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), + copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []), deps = [ ":output_test_helper", "//:benchmark", "//:benchmark_internal_headers", "@com_google_googletest//:gtest", ] + ( - ["@com_google_googletest//:gtest_main"] if (test_src in NEEDS_GTEST_MAIN) else [] + ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else [] ), -# FIXME: Add support for assembly tests to bazel. -# See Issue #556 -# https://github.com/google/benchmark/issues/556 -) for test_src in glob(["*_test.cc"], exclude = ["*_assembly_test.cc"])] + # FIXME: Add support for assembly tests to bazel. + # See Issue #556 + # https://github.com/google/benchmark/issues/556 + ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc"]) +] diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7c6366f789..63c0e58ef3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -162,7 +162,8 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_test(${name} ${name}) endmacro() - add_gtest(statistics_test) + add_gtest(benchmark_gtest) + add_gtest(statistics_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc new file mode 100644 index 0000000000..10683b433a --- /dev/null +++ b/test/benchmark_gtest.cc @@ -0,0 +1,33 @@ +#include + +#include "../src/benchmark_register.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +TEST(AddRangeTest, Simple) { + std::vector dst; + AddRange(&dst, 1, 2, 2); + EXPECT_THAT(dst, testing::ElementsAre(1, 2)); +} + +TEST(AddRangeTest, Simple64) { + std::vector dst; + AddRange(&dst, static_cast(1), static_cast(2), 2); + EXPECT_THAT(dst, testing::ElementsAre(1, 2)); +} + +TEST(AddRangeTest, Advanced) { + std::vector dst; + AddRange(&dst, 5, 15, 2); + EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15)); +} + +TEST(AddRangeTest, Advanced64) { + std::vector dst; + AddRange(&dst, static_cast(5), static_cast(15), 2); + EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15)); +} + +} // end namespace diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index 78802c8da6..3cd4f5565f 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -40,8 +40,8 @@ double CalculatePi(int depth) { return (pi - 1.0) * 4; } -std::set ConstructRandomSet(int size) { - std::set s; +std::set ConstructRandomSet(int64_t size) { + std::set s; for (int i = 0; i < size; ++i) s.insert(s.end(), i); return s; } @@ -64,7 +64,7 @@ BENCHMARK(BM_Factorial)->UseRealTime(); static void BM_CalculatePiRange(benchmark::State& state) { double pi = 0.0; - for (auto _ : state) pi = CalculatePi(state.range(0)); + for (auto _ : state) pi = CalculatePi(static_cast(state.range(0))); std::stringstream ss; ss << pi; state.SetLabel(ss.str()); @@ -74,7 +74,7 @@ BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); static void BM_CalculatePi(benchmark::State& state) { static const int depth = 1024; for (auto _ : state) { - benchmark::DoNotOptimize(CalculatePi(depth)); + benchmark::DoNotOptimize(CalculatePi(static_cast(depth))); } } BENCHMARK(BM_CalculatePi)->Threads(8); @@ -82,7 +82,7 @@ BENCHMARK(BM_CalculatePi)->ThreadRange(1, 32); BENCHMARK(BM_CalculatePi)->ThreadPerCpu(); static void BM_SetInsert(benchmark::State& state) { - std::set data; + std::set data; for (auto _ : state) { state.PauseTiming(); data = ConstructRandomSet(state.range(0)); @@ -103,9 +103,9 @@ static void BM_Sequential(benchmark::State& state) { ValueType v = 42; for (auto _ : state) { Container c; - for (int i = state.range(0); --i;) c.push_back(v); + for (int64_t i = state.range(0); --i;) c.push_back(v); } - const size_t items_processed = state.iterations() * state.range(0); + const int64_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); state.SetBytesProcessed(items_processed * sizeof(v)); } @@ -118,8 +118,9 @@ BENCHMARK_TEMPLATE(BM_Sequential, std::vector, int)->Arg(512); #endif static void BM_StringCompare(benchmark::State& state) { - std::string s1(state.range(0), '-'); - std::string s2(state.range(0), '-'); + size_t len = static_cast(state.range(0)); + std::string s1(len, '-'); + std::string s2(len, '-'); for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); } BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); @@ -154,13 +155,13 @@ static void BM_LongTest(benchmark::State& state) { BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28); static void BM_ParallelMemset(benchmark::State& state) { - int size = state.range(0) / static_cast(sizeof(int)); - int thread_size = size / state.threads; + int64_t size = state.range(0) / static_cast(sizeof(int)); + int thread_size = static_cast(size) / state.threads; int from = thread_size * state.thread_index; int to = from + thread_size; if (state.thread_index == 0) { - test_vector = new std::vector(size); + test_vector = new std::vector(static_cast(size)); } for (auto _ : state) { @@ -178,8 +179,8 @@ static void BM_ParallelMemset(benchmark::State& state) { BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); static void BM_ManualTiming(benchmark::State& state) { - size_t slept_for = 0; - int microseconds = state.range(0); + int64_t slept_for = 0; + int64_t microseconds = state.range(0); std::chrono::duration sleep_duration{ static_cast(microseconds)}; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 89dfa580e6..aa35619def 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -81,9 +81,9 @@ ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); // --------------------------- Testing BigO O(N) --------------------------- // // ========================================================================= // -std::vector ConstructRandomVector(int size) { +std::vector ConstructRandomVector(int64_t size) { std::vector v; - v.reserve(size); + v.reserve(static_cast(size)); for (int i = 0; i < size; ++i) { v.push_back(std::rand() % size); } @@ -92,8 +92,8 @@ std::vector ConstructRandomVector(int size) { void BM_Complexity_O_N(benchmark::State& state) { auto v = ConstructRandomVector(state.range(0)); - const int item_not_in_vector = - state.range(0) * 2; // Test worst case scenario (item not in vector) + // Test worst case scenario (item not in vector) + const int64_t item_not_in_vector = state.range(0) * 2; for (auto _ : state) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); } diff --git a/test/map_test.cc b/test/map_test.cc index 311d2d22b8..dbf7982a36 100644 --- a/test/map_test.cc +++ b/test/map_test.cc @@ -8,7 +8,7 @@ namespace { std::map ConstructRandomMap(int size) { std::map m; for (int i = 0; i < size; ++i) { - m.insert(std::make_pair(rand() % size, rand() % size)); + m.insert(std::make_pair(std::rand() % size, std::rand() % size)); } return m; } @@ -17,14 +17,14 @@ std::map ConstructRandomMap(int size) { // Basic version. static void BM_MapLookup(benchmark::State& state) { - const int size = state.range(0); + const int size = static_cast(state.range(0)); std::map m; for (auto _ : state) { state.PauseTiming(); m = ConstructRandomMap(size); state.ResumeTiming(); for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(rand() % size)); + benchmark::DoNotOptimize(m.find(std::rand() % size)); } } state.SetItemsProcessed(state.iterations() * size); @@ -35,7 +35,7 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); class MapFixture : public ::benchmark::Fixture { public: void SetUp(const ::benchmark::State& st) { - m = ConstructRandomMap(st.range(0)); + m = ConstructRandomMap(static_cast(st.range(0))); } void TearDown(const ::benchmark::State&) { m.clear(); } @@ -44,10 +44,10 @@ class MapFixture : public ::benchmark::Fixture { }; BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { - const int size = state.range(0); + const int size = static_cast(state.range(0)); for (auto _ : state) { for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(rand() % size)); + benchmark::DoNotOptimize(m.find(std::rand() % size)); } } state.SetItemsProcessed(state.iterations() * size); diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc index 0a82382f3c..c64acabc25 100644 --- a/test/multiple_ranges_test.cc +++ b/test/multiple_ranges_test.cc @@ -1,7 +1,9 @@ #include "benchmark/benchmark.h" #include +#include #include +#include class MultipleRangesFixture : public ::benchmark::Fixture { public: @@ -27,25 +29,46 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {7, 6, 3}}) {} void SetUp(const ::benchmark::State& state) { - std::vector ranges = {state.range(0), state.range(1), state.range(2)}; + std::vector ranges = {state.range(0), state.range(1), + state.range(2)}; assert(expectedValues.find(ranges) != expectedValues.end()); actualValues.insert(ranges); } + // NOTE: This is not TearDown as we want to check after _all_ runs are + // complete. virtual ~MultipleRangesFixture() { assert(actualValues.size() == expectedValues.size()); + if (actualValues.size() != expectedValues.size()) { + std::cout << "EXPECTED\n"; + for (auto v : expectedValues) { + std::cout << "{"; + for (int64_t iv : v) { + std::cout << iv << ", "; + } + std::cout << "}\n"; + } + std::cout << "ACTUAL\n"; + for (auto v : actualValues) { + std::cout << "{"; + for (int64_t iv : v) { + std::cout << iv << ", "; + } + std::cout << "}\n"; + } + } } - std::set> expectedValues; - std::set> actualValues; + std::set> expectedValues; + std::set> actualValues; }; BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) { for (auto _ : state) { - int product = state.range(0) * state.range(1) * state.range(2); - for (int x = 0; x < product; x++) { + int64_t product = state.range(0) * state.range(1) * state.range(2); + for (int64_t x = 0; x < product; x++) { benchmark::DoNotOptimize(x); } } diff --git a/test/statistics_test.cc b/test/statistics_gtest.cc similarity index 100% rename from test/statistics_test.cc rename to test/statistics_gtest.cc From 2844167ff99edb63f629c9768ea1f3f94ebfb8fe Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Thu, 5 Apr 2018 18:31:03 -0600 Subject: [PATCH 025/330] Fix #564 - gmock/gmock.h not found in benchmark tests. --- cmake/HandleGTest.cmake | 58 ++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 11528b89fb..3ebe9f32f0 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -54,40 +54,62 @@ macro(build_external_gtest) ) ExternalProject_Get_Property(googletest install_dir) - - add_library(gtest UNKNOWN IMPORTED) - add_library(gtest_main UNKNOWN IMPORTED) + set(GTEST_INCLUDE_DIRS ${install_dir}/include) + file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS}) set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}") - if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG") set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() - file(MAKE_DIRECTORY ${install_dir}/include) - set_target_properties(gtest PROPERTIES - IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest${LIB_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include - ) - set_target_properties(gtest_main PROPERTIES - IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}gtest_main${LIB_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include - ) - add_dependencies(gtest googletest) - add_dependencies(gtest_main googletest) - set(GTEST_BOTH_LIBRARIES gtest gtest_main) - set(GTEST_INCLUDE_DIRS ${install_dir}/include) + + # Use gmock_main instead of gtest_main because it initializes gtest as well. + # Note: The libraries are listed in reverse order of their dependancies. + foreach(LIB gtest gmock gmock_main) + add_library(${LIB} UNKNOWN IMPORTED) + set_target_properties(${LIB} PROPERTIES + IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}${LIB}${LIB_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIRS} + INTERFACE_LINK_LIBRARIES "${GTEST_BOTH_LIBRARIES}" + ) + add_dependencies(${LIB} googletest) + list(APPEND GTEST_BOTH_LIBRARIES ${LIB}) + endforeach() endmacro(build_external_gtest) if (BENCHMARK_ENABLE_GTEST_TESTS) if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest) + set(GTEST_ROOT "${CMAKE_SOURCE_DIR}/googletest") set(INSTALL_GTEST OFF CACHE INTERNAL "") set(INSTALL_GMOCK OFF CACHE INTERNAL "") add_subdirectory(${CMAKE_SOURCE_DIR}/googletest) - set(GTEST_BOTH_LIBRARIES gtest gtest_main) + set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) + foreach(HEADER test mock) + # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we + # have to add the paths ourselves. + set(HFILE g${HEADER}/g${HEADER}.h) + set(HPATH ${GTEST_ROOT}/google${HEADER}/include) + find_path(HEADER_PATH_${HEADER} ${HFILE} + NO_DEFAULT_PATHS + HINTS ${HPATH} + ) + if (NOT HEADER_PATH_${HEADER}) + message(FATAL "Failed to find header ${HFILE} in ${HPATH}") + endif() + list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) + endforeach() elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) build_external_gtest() else() find_package(GTest REQUIRED) + find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h + HINTS ${GTEST_INCLUDE_DIRS}) + if (NOT GMOCK_INCLUDE_DIRS) + message(FATAL "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}") + endif() + set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS}) + # FIXME: We don't currently require the gmock library to build the tests, + # and it's likely we won't find it, so we don't try. As long as we've + # found the gmock/gmock.h header and gtest_main that should be good enough. endif() endif() From 50ffc781b1df6d582d6a453b9942cc8cb512db69 Mon Sep 17 00:00:00 2001 From: Fred Tingaud Date: Mon, 9 Apr 2018 14:40:58 +0200 Subject: [PATCH 026/330] Optimize by using nth_element instead of partial_sort to find the median. (#565) --- src/statistics.cc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/statistics.cc b/src/statistics.cc index d2843679d0..1c91e1015a 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -36,16 +36,19 @@ double StatisticsMean(const std::vector& v) { double StatisticsMedian(const std::vector& v) { if (v.size() < 3) return StatisticsMean(v); - std::vector partial; - // we need roundDown(count/2)+1 slots - partial.resize(1 + (v.size() / 2)); - std::partial_sort_copy(v.begin(), v.end(), partial.begin(), partial.end()); - // did we have odd number of samples? - // if yes, then the last element of partially-sorted vector is the median - // it no, then the average of the last two elements is the median + std::vector copy(v); + + auto center = copy.begin() + v.size() / 2; + std::nth_element(copy.begin(), center, copy.end()); + + // did we have an odd number of samples? + // if yes, then center is the median + // it no, then we are looking for the average between center and the value before if(v.size() % 2 == 1) - return partial.back(); - return (partial[partial.size() - 2] + partial[partial.size() - 1]) / 2.0; + return *center; + auto center2 = copy.begin() + v.size() / 2 - 1; + std::nth_element(copy.begin(), center2, copy.end()); + return (*center + *center2) / 2.0; } // Return the sum of the squares of this sample set From 64e5a13fa05693dc0243817d6081d7c0e5081d6f Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 12 Apr 2018 15:40:24 +0100 Subject: [PATCH 027/330] Ensure 64-bit truncation doesn't happen for complexity_n (#569) * Ensure 64-bit truncation doesn't happen for complexity results * One more complexity_n 64-bit fix * Missed another vector of int * Piping through the int64_t --- include/benchmark/benchmark.h | 4 ++-- src/complexity.cc | 20 ++++++++++---------- src/thread_manager.h | 2 +- test/complexity_test.cc | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index ea9743c685..6be8c12ad4 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -385,7 +385,7 @@ enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(int); +typedef double(BigOFunc)(int64_t); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -1303,7 +1303,7 @@ class BenchmarkReporter { // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; - int complexity_n; + int64_t complexity_n; // what statistics to compute from the measurements const std::vector* statistics; diff --git a/src/complexity.cc b/src/complexity.cc index b1aad695a4..8ae46b6a91 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -28,18 +28,18 @@ namespace benchmark { BigOFunc* FittingCurve(BigO complexity) { switch (complexity) { case oN: - return [](int n) -> double { return n; }; + return [](int64_t n) -> double { return n; }; case oNSquared: - return [](int n) -> double { return std::pow(n, 2); }; + return [](int64_t n) -> double { return std::pow(n, 2); }; case oNCubed: - return [](int n) -> double { return std::pow(n, 3); }; + return [](int64_t n) -> double { return std::pow(n, 3); }; case oLogN: - return [](int n) { return log2(n); }; + return [](int64_t n) { return log2(n); }; case oNLogN: - return [](int n) { return n * log2(n); }; + return [](int64_t n) { return n * log2(n); }; case o1: default: - return [](int) { return 1.0; }; + return [](int64_t) { return 1.0; }; } } @@ -68,12 +68,12 @@ std::string GetBigOString(BigO complexity) { // given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expression (e.g. [](int n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). // For a deeper explanation on the algorithm logic, look the README file at // http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { double sigma_gn = 0.0; @@ -117,7 +117,7 @@ LeastSq MinimalLeastSq(const std::vector& n, // - complexity : If different than oAuto, the fitting curve will stick to // this one. If it is oAuto, it will be calculated the best // fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { CHECK_EQ(n.size(), time.size()); CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two @@ -157,7 +157,7 @@ std::vector ComputeBigO( if (reports.size() < 2) return results; // Accumulators. - std::vector n; + std::vector n; std::vector real_time; std::vector cpu_time; diff --git a/src/thread_manager.h b/src/thread_manager.h index f9ee267dcc..61755a8075 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -40,7 +40,7 @@ class ThreadManager { double manual_time_used = 0; int64_t bytes_processed = 0; int64_t items_processed = 0; - int complexity_n = 0; + int64_t complexity_n = 0; std::string report_label_; std::string error_message_; bool has_error_ = false; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index aa35619def..ab832861ec 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -55,7 +55,7 @@ void BM_Complexity_O1(benchmark::State& state) { } BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int) { +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) { return 1.0; }); @@ -106,7 +106,7 @@ BENCHMARK(BM_Complexity_O_N) BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int n) -> double { return n; }); + ->Complexity([](int64_t n) -> double { return n; }); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -141,7 +141,7 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int n) { return n * log2(n); }); + ->Complexity([](int64_t n) { return n * log2(n); }); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) From c4858d8012acd54d8ef89053c74e9cdcf0fa6649 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 19 Apr 2018 18:40:08 +0100 Subject: [PATCH 028/330] Report the actual iterations run. (#572) Before this change, we would report the number of requested iterations passed to the state. After, we will report the actual number run. As a side-effect, instead of multiplying the expected iterations by the number of threads to get the total number, we can report the actual number of iterations across all threads, which takes into account the situation where some threads might run more iterations than others. --- src/benchmark.cc | 16 ++++++++-------- src/thread_manager.h | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 7b0d1136fc..82b15ac709 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -115,7 +115,7 @@ namespace { BenchmarkReporter::Run CreateRunReport( const benchmark::internal::Benchmark::Instance& b, - const internal::ThreadManager::Result& results, size_t iters, + const internal::ThreadManager::Result& results, double seconds) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -124,8 +124,8 @@ BenchmarkReporter::Run CreateRunReport( report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; - // Report the total iterations across all threads. - report.iterations = static_cast(iters) * b.threads; + // This is the total iterations across all threads. + report.iterations = results.iterations; report.time_unit = b.time_unit; if (!report.error_occurred) { @@ -169,6 +169,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, { MutexLock l(manager->GetBenchmarkMutex()); internal::ThreadManager::Result& results = manager->results; + results.iterations += st.iterations(); results.cpu_time_used += timer.cpu_time_used(); results.real_time_used += timer.real_time_used(); results.manual_time_used += timer.manual_time_used(); @@ -236,18 +237,17 @@ std::vector RunBenchmark( // Determine if this run should be reported; Either it has // run for a sufficient amount of time or because an error was reported. const bool should_report = repetition_num > 0 - || has_explicit_iteration_count // An exact iteration count was requested + || has_explicit_iteration_count // An exact iteration count was requested || results.has_error_ - || iters >= kMaxIterations - || seconds >= min_time // the elapsed time is large enough + || iters >= kMaxIterations // No chance to try again, we hit the limit. + || seconds >= min_time // the elapsed time is large enough // CPU time is specified but the elapsed real time greatly exceeds the // minimum time. Note that user provided timers are except from this // sanity check. || ((results.real_time_used >= 5 * min_time) && !b.use_manual_time); if (should_report) { - BenchmarkReporter::Run report = - CreateRunReport(b, results, iters, seconds); + BenchmarkReporter::Run report = CreateRunReport(b, results, seconds); if (!report.error_occurred && b.complexity != oNone) complexity_reports->push_back(report); reports.push_back(report); diff --git a/src/thread_manager.h b/src/thread_manager.h index 61755a8075..82b4d72b62 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -1,6 +1,9 @@ #ifndef BENCHMARK_THREAD_MANAGER_H #define BENCHMARK_THREAD_MANAGER_H +#include + +#include "benchmark/benchmark.h" #include "mutex.h" namespace benchmark { @@ -35,6 +38,7 @@ class ThreadManager { public: struct Result { + int64_t iterations = 0; double real_time_used = 0; double cpu_time_used = 0; double manual_time_used = 0; From 64d4805dd7861fa265e3d8f10efc056985d4e945 Mon Sep 17 00:00:00 2001 From: Victor Costan Date: Mon, 23 Apr 2018 03:58:02 -0700 Subject: [PATCH 029/330] Fix precision loss warning in MSVC. (#574) --- src/complexity.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/complexity.cc b/src/complexity.cc index 8ae46b6a91..97bf6e09b3 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -28,7 +28,7 @@ namespace benchmark { BigOFunc* FittingCurve(BigO complexity) { switch (complexity) { case oN: - return [](int64_t n) -> double { return n; }; + return [](int64_t n) -> double { return static_cast(n); }; case oNSquared: return [](int64_t n) -> double { return std::pow(n, 2); }; case oNCubed: From 105ac14b2f671f7d448f621290404098189f424d Mon Sep 17 00:00:00 2001 From: Yangqing Jia Date: Mon, 23 Apr 2018 12:57:03 -0700 Subject: [PATCH 030/330] Add caching for cxx_feature_check (#573) --- cmake/CXXFeatureCheck.cmake | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index b2a8217157..c4c4d660f1 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -27,25 +27,27 @@ function(cxx_feature_check FILE) return() endif() - message("-- Performing Test ${FEATURE}") - if(CMAKE_CROSSCOMPILING) - try_compile(COMPILE_${FEATURE} - ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) - if(COMPILE_${FEATURE}) - message(WARNING - "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") - set(RUN_${FEATURE} 0) + if (NOT DEFINED COMPILE_${FEATURE}) + message("-- Performing Test ${FEATURE}") + if(CMAKE_CROSSCOMPILING) + try_compile(COMPILE_${FEATURE} + ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp + CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + if(COMPILE_${FEATURE}) + message(WARNING + "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") + set(RUN_${FEATURE} 0) + else() + set(RUN_${FEATURE} 1) + endif() else() - set(RUN_${FEATURE} 1) + message("-- Performing Test ${FEATURE}") + try_run(RUN_${FEATURE} COMPILE_${FEATURE} + ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp + CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) endif() - else() - message("-- Performing Test ${FEATURE}") - try_run(RUN_${FEATURE} COMPILE_${FEATURE} - ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) endif() if(RUN_${FEATURE} EQUAL 0) From ed1bac8434b3f970a5b7de3fea8f09dc4b1773ee Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 26 Apr 2018 10:56:06 +0100 Subject: [PATCH 031/330] Issue 571: Allow support for negative regex filtering (#576) * Allow support for negative regex filtering This patch allows one to apply a negation to the entire regex filter by appending it with a '-' character, much in the same style as GoogleTest uses. * Address issues in PR * Add unit tests for negative filtering --- src/benchmark_register.cc | 12 +++++++++--- test/CMakeLists.txt | 9 +++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 4fea6d915f..dc6f935685 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -77,7 +77,7 @@ class BenchmarkFamilies { // Extract the list of benchmark instances that match the specified // regular expression. - bool FindBenchmarks(const std::string& re, + bool FindBenchmarks(std::string re, std::vector* benchmarks, std::ostream* Err); @@ -107,13 +107,18 @@ void BenchmarkFamilies::ClearBenchmarks() { } bool BenchmarkFamilies::FindBenchmarks( - const std::string& spec, std::vector* benchmarks, + std::string spec, std::vector* benchmarks, std::ostream* ErrStream) { CHECK(ErrStream); auto& Err = *ErrStream; // Make regular expression out of command-line flag std::string error_msg; Regex re; + bool isNegativeFilter = false; + if(spec[0] == '-') { + spec.replace(0, 1, ""); + isNegativeFilter = true; + } if (!re.Init(spec, &error_msg)) { Err << "Could not compile benchmark re: " << error_msg << std::endl; return false; @@ -199,7 +204,8 @@ bool BenchmarkFamilies::FindBenchmarks( instance.name += StrFormat("/threads:%d", instance.threads); } - if (re.Match(instance.name)) { + if ((re.Match(instance.name) && !isNegativeFilter) || + (!re.Match(instance.name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 63c0e58ef3..287e22851a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,14 +59,23 @@ macro(add_filter_test name filter expect) endmacro(add_filter_test) add_filter_test(filter_simple "Foo" 3) +add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) +add_filter_test(filter_suffix_negative "-BM_.*" 1) add_filter_test(filter_regex_all ".*" 5) +add_filter_test(filter_regex_all_negative "-.*" 0) add_filter_test(filter_regex_blank "" 5) +add_filter_test(filter_regex_blank_negative "-" 0) add_filter_test(filter_regex_none "monkey" 0) +add_filter_test(filter_regex_none_negative "-monkey" 5) add_filter_test(filter_regex_wildcard ".*Foo.*" 3) +add_filter_test(filter_regex_wildcard_negative "-.*Foo.*" 2) add_filter_test(filter_regex_begin "^BM_.*" 4) +add_filter_test(filter_regex_begin_negative "-^BM_.*" 1) add_filter_test(filter_regex_begin2 "^N" 1) +add_filter_test(filter_regex_begin2_negative "-^N" 4) add_filter_test(filter_regex_end ".*Ba$" 1) +add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) add_test(options_benchmarks options_test --benchmark_min_time=0.01) From b678a2026df75bdcaebbda40440b8495babf078e Mon Sep 17 00:00:00 2001 From: Alex Strelnikov Date: Tue, 1 May 2018 10:24:38 -0400 Subject: [PATCH 032/330] Fix bazel config to link against pthread. (#579) The benchmarks in the test/ currently build because they all include a dep on gtest, which brings in pthread when needed. --- BUILD.bazel | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 35605ccb91..7a9bac7a40 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,5 +1,13 @@ licenses(["notice"]) +config_setting( + name = "windows", + values = { + "cpu": "x64_windows", + }, + visibility = [":__subpackages__"], +) + cc_library( name = "benchmark", srcs = glob([ @@ -7,6 +15,10 @@ cc_library( "src/*.h", ]), hdrs = ["include/benchmark/benchmark.h"], + linkopts = select({ + ":windows": [], + "//conditions:default": ["-pthread"], + }), strip_include_prefix = "include", visibility = ["//visibility:public"], ) From 62a9d756eacf28b5be59429d25f1fae3e202e5e3 Mon Sep 17 00:00:00 2001 From: Alex Strelnikov Date: Wed, 2 May 2018 06:23:18 -0400 Subject: [PATCH 033/330] Update bazel WORKSPACE and BUILD files to work better on Windows. (#581) Note, bazel only supports MSVC on Windows, and not MinGW, so linking against shlwapi.lib only needs to follow MSVC conventions. git_repository() did not work in local testing, so is swapped for http_archive(). The latter is also documented as the preferred way to depend on an external library in bazel. --- BUILD.bazel | 2 +- README.md | 4 ++-- WORKSPACE | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 7a9bac7a40..76b85d88e6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,7 +16,7 @@ cc_library( ]), hdrs = ["include/benchmark/benchmark.h"], linkopts = select({ - ":windows": [], + ":windows": ["-DEFAULTLIB:shlwapi.lib"], "//conditions:default": ["-pthread"], }), strip_include_prefix = "include", diff --git a/README.md b/README.md index c8f8c01f0f..dbf59d7f5a 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ We need to install the library globally now sudo make install ``` -Now you have google/benchmark installed in your machine +Now you have google/benchmark installed in your machine Note: Don't forget to link to pthread library while building ## Stable and Experimental Library Versions @@ -933,7 +933,7 @@ sudo cpupower frequency-set --governor powersave # Known Issues -### Windows +### Windows with CMake * Users must manually link `shlwapi.lib`. Failure to do so may result in unresolved symbols. diff --git a/WORKSPACE b/WORKSPACE index 9ba32fd75e..54734f1ea5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,7 @@ workspace(name = "com_github_google_benchmark") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -git_repository( - name = "com_google_googletest", - commit = "3f0cf6b62ad1eb50d8736538363d3580dd640c3e", # HEAD - remote = "https://github.com/google/googletest", +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], + strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", ) From ea5551e7b3129d8ee468b15e2a4242edd610ee02 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Wed, 2 May 2018 18:26:43 +0800 Subject: [PATCH 034/330] Porting into OpenBSD (#582) --- src/internal_macros.h | 2 ++ src/sysinfo.cc | 30 ++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/internal_macros.h b/src/internal_macros.h index 248ef26259..f7b9203e53 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -51,6 +51,8 @@ #define BENCHMARK_OS_FREEBSD 1 #elif defined(__NetBSD__) #define BENCHMARK_OS_NETBSD 1 +#elif defined(__OpenBSD__) + #define BENCHMARK_OS_OPENBSD 1 #elif defined(__linux__) #define BENCHMARK_OS_LINUX 1 #elif defined(__native_client__) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index dab020b535..d19d0ef4c1 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -28,7 +28,7 @@ #include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include #if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX || \ - defined BENCHMARK_OS_NETBSD + defined BENCHMARK_OS_NETBSD || defined BENCHMARK_OS_OPENBSD #define BENCHMARK_HAS_SYSCTL #include #endif @@ -136,6 +136,26 @@ struct ValueUnion { }; ValueUnion GetSysctlImp(std::string const& Name) { +#if defined BENCHMARK_OS_OPENBSD + int mib[2]; + + mib[0] = CTL_HW; + if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")){ + ValueUnion buff(sizeof(int)); + + if (Name == "hw.ncpu") { + mib[1] = HW_NCPU; + } else { + mib[1] = HW_CPUSPEED; + } + + if (sysctl(mib, 2, buff.data(), &buff.Size, nullptr, 0) == -1) { + return ValueUnion(); + } + return buff; + } + return ValueUnion(); +#else size_t CurBuffSize = 0; if (sysctlbyname(Name.c_str(), nullptr, &CurBuffSize, nullptr, 0) == -1) return ValueUnion(); @@ -144,6 +164,7 @@ ValueUnion GetSysctlImp(std::string const& Name) { if (sysctlbyname(Name.c_str(), buff.data(), &buff.Size, nullptr, 0) == 0) return buff; return ValueUnion(); +#endif } BENCHMARK_MAYBE_UNUSED @@ -488,12 +509,17 @@ double GetCPUCyclesPerSecond() { constexpr auto* FreqStr = #if defined(BENCHMARK_OS_FREEBSD) || defined(BENCHMARK_OS_NETBSD) "machdep.tsc_freq"; +#elif defined BENCHMARK_OS_OPENBSD + "hw.cpuspeed"; #else "hw.cpufrequency"; #endif unsigned long long hz = 0; +#if defined BENCHMARK_OS_OPENBSD + if (GetSysctl(FreqStr, &hz)) return hz * 1000000; +#else if (GetSysctl(FreqStr, &hz)) return hz; - +#endif fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", FreqStr, strerror(errno)); From 8986839e4ac67facb52abc7fff3d5082e854aab5 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 3 May 2018 01:34:26 -0700 Subject: [PATCH 035/330] Use __EMSCRIPTEN__ (rather then EMSCRIPTEN) to check for emscripten (#583) The old EMSCRIPTEN macro is deprecated and not enabled when EMCC_STRICT is set. Also fix a typo in EMSCRIPTN (not sure how this ever worked). --- include/benchmark/benchmark.h | 2 +- src/internal_macros.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 6be8c12ad4..a6015b8164 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -292,7 +292,7 @@ BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); #if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ - defined(EMSCRIPTN) + defined(__EMSCRIPTEN__) # define BENCHMARK_HAS_NO_INLINE_ASSEMBLY #endif diff --git a/src/internal_macros.h b/src/internal_macros.h index f7b9203e53..edb8a5c0a3 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -57,7 +57,7 @@ #define BENCHMARK_OS_LINUX 1 #elif defined(__native_client__) #define BENCHMARK_OS_NACL 1 -#elif defined(EMSCRIPTEN) +#elif defined(__EMSCRIPTEN__) #define BENCHMARK_OS_EMSCRIPTEN 1 #elif defined(__rtems__) #define BENCHMARK_OS_RTEMS 1 From 16af64500fe13d63a5ad57e81f05cb0b9383875d Mon Sep 17 00:00:00 2001 From: php1ic Date: Tue, 8 May 2018 11:29:09 +0100 Subject: [PATCH 036/330] Run git from the source directory (#589) (#590) Git was being executed in the current directory, so could not get the latest tag if cmake was run from a build directory. Force git to be run from with the source directory. --- cmake/GetGitVersion.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/GetGitVersion.cmake b/cmake/GetGitVersion.cmake index 8dd9480045..88cebe3a1c 100644 --- a/cmake/GetGitVersion.cmake +++ b/cmake/GetGitVersion.cmake @@ -21,6 +21,7 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) execute_process(COMMAND ${GIT_EXECUTABLE} describe --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status OUTPUT_VARIABLE GIT_VERSION ERROR_QUIET) @@ -33,9 +34,11 @@ function(get_git_version var) # Work out if the repository is dirty execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_QUIET ERROR_QUIET) execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_DIFF_INDEX ERROR_QUIET) string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) From e8ddd907bb6a881f40069ea73caa74cb9112c564 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 8 May 2018 13:33:37 +0300 Subject: [PATCH 037/330] There is no "FATAL" in message(), only "FATAL_ERROR" (#584) --- CMakeLists.txt | 2 +- cmake/HandleGTest.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c107937ef..b1c1d3d5a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -216,7 +216,7 @@ if (BENCHMARK_USE_LIBCXX) # linker flags appear before all linker inputs and -lc++ must appear after. list(APPEND BENCHMARK_CXX_LIBRARIES c++) else() - message(FATAL "-DBENCHMARK_USE_LIBCXX:BOOL=ON is not supported for compiler") + message(FATAL_ERROR "-DBENCHMARK_USE_LIBCXX:BOOL=ON is not supported for compiler") endif() endif(BENCHMARK_USE_LIBCXX) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 3ebe9f32f0..43c71bf9d3 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -94,7 +94,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) HINTS ${HPATH} ) if (NOT HEADER_PATH_${HEADER}) - message(FATAL "Failed to find header ${HFILE} in ${HPATH}") + message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") endif() list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) endforeach() @@ -105,7 +105,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h HINTS ${GTEST_INCLUDE_DIRS}) if (NOT GMOCK_INCLUDE_DIRS) - message(FATAL "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}") + message(FATAL_ERROR "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}") endif() set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS}) # FIXME: We don't currently require the gmock library to build the tests, From 718cc91d004e15ce483cb1ba9bcd5e3a1f025184 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 8 May 2018 13:34:31 +0300 Subject: [PATCH 038/330] [Tools] Fix a few python3-compatibility issues (#585) --- tools/compare.py | 8 ++++++-- tools/gbench/report.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/compare.py b/tools/compare.py index c4a47e8d50..f0a4455f5f 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -138,6 +138,9 @@ def main(): # Parse the command line flags parser = create_parser() args, unknown_args = parser.parse_known_args() + if args.mode is None: + parser.print_help() + exit(1) assert not unknown_args benchmark_options = args.benchmark_options @@ -175,6 +178,7 @@ def main(): else: # should never happen print("Unrecognized mode of operation: '%s'" % args.mode) + parser.print_help() exit(1) check_inputs(test_baseline, test_contender, benchmark_options) @@ -218,8 +222,8 @@ def setUp(self): os.path.realpath(__file__)), 'gbench', 'Inputs') - self.testInput0 = os.path.join(testInputs, 'test_baseline_run1.json') - self.testInput1 = os.path.join(testInputs, 'test_baseline_run2.json') + self.testInput0 = os.path.join(testInputs, 'test1_run1.json') + self.testInput1 = os.path.join(testInputs, 'test1_run2.json') def test_benchmarks_basic(self): parsed = self.parser.parse_args( diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 8d68fe96ee..0c090981a8 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -191,7 +191,7 @@ def test_basic(self): json2 = filter_benchmark(json, "BM_O.e", ".") output_lines_with_header = generate_difference_report(json1, json2, use_color=False) output_lines = output_lines_with_header[2:] - print "\n" + print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) for i in range(0, len(output_lines)): From e90801ae475f23877319d67b96c07bf0c52405f3 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Wed, 9 May 2018 17:31:24 +0800 Subject: [PATCH 039/330] Remove unnecessary memset functions. (#591) --- src/timers.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index f98b98424d..2010e2450b 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -193,7 +193,6 @@ std::string DateTimeString(bool local) { std::strftime(storage, sizeof(storage), "%x %X", ::localtime(&now)); #else std::tm timeinfo; - std::memset(&timeinfo, 0, sizeof(std::tm)); ::localtime_r(&now, &timeinfo); written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); #endif @@ -202,7 +201,6 @@ std::string DateTimeString(bool local) { written = std::strftime(storage, sizeof(storage), "%x %X", ::gmtime(&now)); #else std::tm timeinfo; - std::memset(&timeinfo, 0, sizeof(std::tm)); ::gmtime_r(&now, &timeinfo); written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); #endif From 6d74c0625b8e88c1afce72b4f383c91b9a99dbe6 Mon Sep 17 00:00:00 2001 From: Deniz Evrenci Date: Mon, 14 May 2018 23:02:49 +0900 Subject: [PATCH 040/330] split_list is not defined for assembly tests (#595) * Update AUTHORS and CONTRIBUTORS * split_list is not defined for assembly tests --- AUTHORS | 1 + CONTRIBUTORS | 1 + cmake/HandleGTest.cmake | 4 +--- cmake/split_list.cmake | 3 +++ test/AssemblyTests.cmake | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 cmake/split_list.cmake diff --git a/AUTHORS b/AUTHORS index 45adb27ee5..f8219036d2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Arne Beer Carto Christopher Seymour David Coeurjolly +Deniz Evrenci Dirac Research Dominik Czarnota Eric Fiselier diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 2f1999be74..1cf04db17e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -28,6 +28,7 @@ Billy Robert O'Neal III Chris Kennelly Christopher Seymour David Coeurjolly +Deniz Evrenci Dominic Hamon Dominik Czarnota Eric Fiselier diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 43c71bf9d3..7ce1a633d6 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -1,7 +1,5 @@ -macro(split_list listname) - string(REPLACE ";" " " ${listname} "${${listname}}") -endmacro() +include(split_list) macro(build_external_gtest) include(ExternalProject) diff --git a/cmake/split_list.cmake b/cmake/split_list.cmake new file mode 100644 index 0000000000..67aed3fdc8 --- /dev/null +++ b/cmake/split_list.cmake @@ -0,0 +1,3 @@ +macro(split_list listname) + string(REPLACE ";" " " ${listname} "${${listname}}") +endmacro() diff --git a/test/AssemblyTests.cmake b/test/AssemblyTests.cmake index d8f321aa61..3d078586f1 100644 --- a/test/AssemblyTests.cmake +++ b/test/AssemblyTests.cmake @@ -1,4 +1,5 @@ +include(split_list) set(ASM_TEST_FLAGS "") check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) From ce3fde16cb4e3949f8d7c20f49867023af92a6e1 Mon Sep 17 00:00:00 2001 From: Samuel Panzer Date: Thu, 24 May 2018 02:33:19 -0700 Subject: [PATCH 041/330] Return 0 from State::iterations() when not yet started. (#598) * Return a reasonable value from State::iterations() even before starting a benchmark * Optimize State::iterations() for started case. --- include/benchmark/benchmark.h | 5 ++++- test/basic_test.cc | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index a6015b8164..23dd3d09b1 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -575,7 +575,10 @@ class State { BENCHMARK_ALWAYS_INLINE size_t iterations() const { - return (max_iterations - total_iterations_ + batch_leftover_); + if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { + return 0; + } + return max_iterations - total_iterations_ + batch_leftover_; } private: // items we expect on the first cache line (ie 64 bytes of the struct) diff --git a/test/basic_test.cc b/test/basic_test.cc index 100f68985c..d07fbc00b1 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -99,6 +99,7 @@ BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); void BM_KeepRunning(benchmark::State& state) { size_t iter_count = 0; + assert(iter_count == state.iterations()); while (state.KeepRunning()) { ++iter_count; } From d7aed73677888834d4e8af2b300d01bfb724b70f Mon Sep 17 00:00:00 2001 From: mattreecebentley Date: Thu, 24 May 2018 21:50:35 +1200 Subject: [PATCH 042/330] Corrections, additions to initial doc (#600) * Correct/clarify build/install instructions GTest is google test, don't obsfucate needlessly for newcomers. Adding google test into installation guide helps newcomers. Third option under this line: "Note that Google Benchmark requires Google Test to build and run the tests. This dependency can be provided three ways:" Was not true (did not occur). If there is a further option that needs to be specified in order for that functionality to work it needs to be specified. * Add prerequisite knowledge section A lot of assumptions are made about the reader in the documentation. This is unfortunate. * Removal of abbreviations for google test --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dbf59d7f5a..b2e8eb648c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The basic steps for configuring and building the library look like this: ```bash $ git clone https://github.com/google/benchmark.git -# Benchmark requires GTest as a dependency. Add the source tree as a subdirectory. +# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. $ git clone https://github.com/google/googletest.git benchmark/googletest $ mkdir build && cd build $ cmake -G [options] ../benchmark @@ -31,15 +31,13 @@ $ cmake -G [options] ../benchmark $ make ``` -Note that Google Benchmark requires GTest to build and run the tests. This -dependency can be provided three ways: +Note that Google Benchmark requires Google Test to build and run the tests. This +dependency can be provided two ways: -* Checkout the GTest sources into `benchmark/googletest`. +* Checkout the Google Test sources into `benchmark/googletest` as above. * Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during configuration, the library will automatically download and build any required dependencies. -* Otherwise, if nothing is done, CMake will use `find_package(GTest REQUIRED)` - to resolve the required GTest dependency. If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF` to `CMAKE_ARGS`. @@ -61,6 +59,7 @@ Now, let's clone the repository and build it ``` git clone https://github.com/google/benchmark.git cd benchmark +git clone https://github.com/google/googletest.git mkdir build cd build cmake .. -DCMAKE_BUILD_TYPE=RELEASE @@ -88,6 +87,11 @@ to use, test, and provide feedback on the new features are encouraged to try this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. +##Prerequisite knowledge + +Before attempting to understand this framework one should ideally have some familiarity with the structure and format of the Google Test framework, upon which it is based. Documentation for Google Test, including a "Getting Started" (primer) guide, is available here: +https://github.com/google/googletest/blob/master/googletest/docs/Documentation.md + ## Example usage ### Basic usage From e776aa0275e293707b6a0901e0e8d8a8a3679508 Mon Sep 17 00:00:00 2001 From: Alex Strelnikov Date: Fri, 25 May 2018 06:18:58 -0400 Subject: [PATCH 043/330] Add benchmark_main target. (#601) * Add benchmark_main library with support for Bazel. * fix newline at end of file * Add CMake support for benchmark_main. * Mention optionally using benchmark_main in README. --- BUILD.bazel | 20 ++++++++++++++++---- README.md | 5 ++++- src/CMakeLists.txt | 15 ++++++++++++++- src/benchmark_main.cc | 17 +++++++++++++++++ test/BUILD | 10 +++++++++- test/CMakeLists.txt | 7 +++++++ test/link_main_test.cc | 8 ++++++++ 7 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 src/benchmark_main.cc create mode 100644 test/link_main_test.cc diff --git a/BUILD.bazel b/BUILD.bazel index 76b85d88e6..6ee69f2907 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,10 +10,13 @@ config_setting( cc_library( name = "benchmark", - srcs = glob([ - "src/*.cc", - "src/*.h", - ]), + srcs = glob( + [ + "src/*.cc", + "src/*.h", + ], + exclude = ["src/benchmark_main.cc"], + ), hdrs = ["include/benchmark/benchmark.h"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], @@ -23,6 +26,15 @@ cc_library( visibility = ["//visibility:public"], ) +cc_library( + name = "benchmark_main", + srcs = ["src/benchmark_main.cc"], + hdrs = ["include/benchmark/benchmark.h"], + strip_include_prefix = "include", + visibility = ["//visibility:public"], + deps = [":benchmark"], +) + cc_library( name = "benchmark_internal_headers", hdrs = glob(["src/*.h"]), diff --git a/README.md b/README.md index b2e8eb648c..0341c31bd7 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,10 @@ BENCHMARK(BM_StringCopy); BENCHMARK_MAIN(); ``` -Don't forget to inform your linker to add benchmark library e.g. through `-lbenchmark` compilation flag. +Don't forget to inform your linker to add benchmark library e.g. through +`-lbenchmark` compilation flag. Alternatively, you may leave out the +`BENCHMARK_MAIN();` at the end of the source file and link against +`-lbenchmark_main` to get the same default behavior. The benchmark library will reporting the timing for the code within the `for(...)` loop. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 836549e3ca..701804ba0e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,6 +11,7 @@ file(GLOB *.cc ${PROJECT_SOURCE_DIR}/include/benchmark/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.h) +list(FILTER SOURCE_FILES EXCLUDE REGEX "benchmark_main\\.cc") add_library(benchmark ${SOURCE_FILES}) set_target_properties(benchmark PROPERTIES @@ -39,6 +40,18 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") target_link_libraries(benchmark kstat) endif() +# Benchmark main library +add_library(benchmark_main "benchmark_main.cc") +set_target_properties(benchmark_main PROPERTIES + OUTPUT_NAME "benchmark_main" + VERSION ${GENERIC_LIB_VERSION} + SOVERSION ${GENERIC_LIB_SOVERSION} +) +target_include_directories(benchmark PUBLIC + $ + ) +target_link_libraries(benchmark_main benchmark) + set(include_install_dir "include") set(lib_install_dir "lib/") set(bin_install_dir "bin/") @@ -65,7 +78,7 @@ configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}" @ON if (BENCHMARK_ENABLE_INSTALL) # Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable) install( - TARGETS benchmark + TARGETS benchmark benchmark_main EXPORT ${targets_export_name} ARCHIVE DESTINATION ${lib_install_dir} LIBRARY DESTINATION ${lib_install_dir} diff --git a/src/benchmark_main.cc b/src/benchmark_main.cc new file mode 100644 index 0000000000..b3b2478314 --- /dev/null +++ b/src/benchmark_main.cc @@ -0,0 +1,17 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" + +BENCHMARK_MAIN(); diff --git a/test/BUILD b/test/BUILD index 2b3a391296..3f174c486f 100644 --- a/test/BUILD +++ b/test/BUILD @@ -53,5 +53,13 @@ cc_library( # FIXME: Add support for assembly tests to bazel. # See Issue #556 # https://github.com/google/benchmark/issues/556 - ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc"]) + ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc", "link_main_test.cc"]) ] + +cc_test( + name = "link_main_test", + size = "small", + srcs = ["link_main_test.cc"], + copts = TEST_COPTS, + deps = ["//:benchmark_main"], +) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 287e22851a..05ae804bfe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,6 +41,10 @@ macro(compile_benchmark_test name) target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_benchmark_test) +macro(compile_benchmark_test_with_main name) + add_executable(${name} "${name}.cc") + target_link_libraries(${name} benchmark_main) +endmacro(compile_benchmark_test_with_main) macro(compile_output_test name) add_executable(${name} "${name}.cc" output_test.h) @@ -109,6 +113,9 @@ add_test(map_test map_test --benchmark_min_time=0.01) compile_benchmark_test(multiple_ranges_test) add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01) +compile_benchmark_test_with_main(link_main_test) +add_test(link_main_test link_main_test --benchmark_min_time=0.01) + compile_output_test(reporter_output_test) add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) diff --git a/test/link_main_test.cc b/test/link_main_test.cc new file mode 100644 index 0000000000..241ad5c390 --- /dev/null +++ b/test/link_main_test.cc @@ -0,0 +1,8 @@ +#include "benchmark/benchmark.h" + +void BM_empty(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(state.iterations()); + } +} +BENCHMARK(BM_empty); From ec0f69c28e412ec1fb1e8d170ada4faeebdc8293 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 29 May 2018 10:36:54 +0100 Subject: [PATCH 044/330] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0341c31bd7..749039a0ac 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ to use, test, and provide feedback on the new features are encouraged to try this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. -##Prerequisite knowledge +## Prerequisite knowledge Before attempting to understand this framework one should ideally have some familiarity with the structure and format of the Google Test framework, upon which it is based. Documentation for Google Test, including a "Getting Started" (primer) guide, is available here: https://github.com/google/googletest/blob/master/googletest/docs/Documentation.md From a6a1b0d765b116bb9c777d45a299ae84a2760981 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 29 May 2018 13:13:28 +0300 Subject: [PATCH 045/330] Benchmarking is hard. Making sense of the benchmarking results is even harder. (#593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first problem you have to solve yourself. The second one can be aided. The benchmark library can compute some statistics over the repetitions, which helps with grasping the results somewhat. But that is only for the one set of results. It does not really help to compare the two benchmark results, which is the interesting bit. Thankfully, there are these bundled `tools/compare.py` and `tools/compare_bench.py` scripts. They can provide a diff between two benchmarking results. Yay! Except not really, it's just a diff, while it is very informative and better than nothing, it does not really help answer The Question - am i just looking at the noise? It's like not having these per-benchmark statistics... Roughly, we can formulate the question as: > Are these two benchmarks the same? > Did my change actually change anything, or is the difference below the noise level? Well, this really sounds like a [null hypothesis](https://en.wikipedia.org/wiki/Null_hypothesis), does it not? So maybe we can use statistics here, and solve all our problems? lol, no, it won't solve all the problems. But maybe it will act as a tool, to better understand the output, just like the usual statistics on the repetitions... I'm making an assumption here that most of the people care about the change of average value, not the standard deviation. Thus i believe we can use T-Test, be it either [Student's t-test](https://en.wikipedia.org/wiki/Student%27s_t-test), or [Welch's t-test](https://en.wikipedia.org/wiki/Welch%27s_t-test). **EDIT**: however, after @dominichamon review, it was decided that it is better to use more robust [Mann–Whitney U test](https://en.wikipedia.org/wiki/Mann–Whitney_U_test) I'm using [scipy.stats.mannwhitneyu](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mannwhitneyu.html#scipy.stats.mannwhitneyu). There are two new user-facing knobs: ``` $ ./compare.py --help usage: compare.py [-h] [-u] [--alpha UTEST_ALPHA] {benchmarks,filters,benchmarksfiltered} ... versatile benchmark output compare tool <...> optional arguments: -h, --help show this help message and exit -u, --utest Do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample. WARNING: requires **LARGE** (9 or more) number of repetitions to be meaningful! --alpha UTEST_ALPHA significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected. (default: 0.0500) ``` Example output: ![screenshot_20180512_175517](https://user-images.githubusercontent.com/88600/39958581-ae897924-560d-11e8-81b9-806db6c3e691.png) As you can guess, the alpha does affect anything but the coloring of the computed p-values. If it is green, then the change in the average values is statistically-significant. I'm detecting the repetitions by matching name. This way, no changes to the json are _needed_. Caveats: * This won't work if the json is not in the same order as outputted by the benchmark, or if the parsing does not retain the ordering. * This won't work if after the grouped repetitions there isn't at least one row with different name (e.g. statistic). Since there isn't a knob to disable printing of statistics (only the other way around), i'm not too worried about this. * **The results will be wrong if the repetition count is different between the two benchmarks being compared.** * Even though i have added (hopefully full) test coverage, the code of these python tools is staring to look a bit jumbled. * So far i have added this only to the `tools/compare.py`. Should i add it to `tools/compare_bench.py` too? Or should we deduplicate them (by removing the latter one)? --- tools/compare.py | 62 +++++++++++- tools/gbench/Inputs/test3_run0.json | 39 ++++++++ tools/gbench/Inputs/test3_run1.json | 39 ++++++++ tools/gbench/report.py | 150 ++++++++++++++++++++++++---- 4 files changed, 270 insertions(+), 20 deletions(-) create mode 100644 tools/gbench/Inputs/test3_run0.json create mode 100644 tools/gbench/Inputs/test3_run1.json diff --git a/tools/compare.py b/tools/compare.py index f0a4455f5f..f293306eac 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -35,6 +35,22 @@ def check_inputs(in1, in2, flags): def create_parser(): parser = ArgumentParser( description='versatile benchmark output compare tool') + + utest = parser.add_argument_group() + utest.add_argument( + '-u', + '--utest', + action="store_true", + help="Do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than 9) number of repetitions to be meaningful!") + alpha_default = 0.05 + utest.add_argument( + "--alpha", + dest='utest_alpha', + default=alpha_default, + type=float, + help=("significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)") % + alpha_default) + subparsers = parser.add_subparsers( help='This tool has multiple modes of operation:', dest='mode') @@ -139,8 +155,8 @@ def main(): parser = create_parser() args, unknown_args = parser.parse_known_args() if args.mode is None: - parser.print_help() - exit(1) + parser.print_help() + exit(1) assert not unknown_args benchmark_options = args.benchmark_options @@ -205,7 +221,8 @@ def main(): json2_orig, filter_contender, replacement) # Diff and output - output_lines = gbench.report.generate_difference_report(json1, json2) + output_lines = gbench.report.generate_difference_report( + json1, json2, args.utest, args.utest_alpha) print(description) for ln in output_lines: print(ln) @@ -228,6 +245,37 @@ def setUp(self): def test_benchmarks_basic(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.utest) + self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.test_baseline[0].name, self.testInput0) + self.assertEqual(parsed.test_contender[0].name, self.testInput1) + self.assertFalse(parsed.benchmark_options) + + def test_benchmarks_basic_with_utest(self): + parsed = self.parser.parse_args( + ['-u', 'benchmarks', self.testInput0, self.testInput1]) + self.assertTrue(parsed.utest) + self.assertEqual(parsed.utest_alpha, 0.05) + self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.test_baseline[0].name, self.testInput0) + self.assertEqual(parsed.test_contender[0].name, self.testInput1) + self.assertFalse(parsed.benchmark_options) + + def test_benchmarks_basic_with_utest(self): + parsed = self.parser.parse_args( + ['--utest', 'benchmarks', self.testInput0, self.testInput1]) + self.assertTrue(parsed.utest) + self.assertEqual(parsed.utest_alpha, 0.05) + self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.test_baseline[0].name, self.testInput0) + self.assertEqual(parsed.test_contender[0].name, self.testInput1) + self.assertFalse(parsed.benchmark_options) + + def test_benchmarks_basic_with_utest_alpha(self): + parsed = self.parser.parse_args( + ['--utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + self.assertTrue(parsed.utest) + self.assertEqual(parsed.utest_alpha, 0.314) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) @@ -236,6 +284,7 @@ def test_benchmarks_basic(self): def test_benchmarks_with_remainder(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, 'd']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) @@ -244,6 +293,7 @@ def test_benchmarks_with_remainder(self): def test_benchmarks_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, '--', 'e']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) @@ -252,6 +302,7 @@ def test_benchmarks_with_remainder_after_doubleminus(self): def test_filters_basic(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -261,6 +312,7 @@ def test_filters_basic(self): def test_filters_with_remainder(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', 'e']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -270,6 +322,7 @@ def test_filters_with_remainder(self): def test_filters_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', '--', 'f']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -279,6 +332,7 @@ def test_filters_with_remainder_after_doubleminus(self): def test_benchmarksfiltered_basic(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -289,6 +343,7 @@ def test_benchmarksfiltered_basic(self): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -299,6 +354,7 @@ def test_benchmarksfiltered_with_remainder(self): def test_benchmarksfiltered_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g']) + self.assertFalse(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') diff --git a/tools/gbench/Inputs/test3_run0.json b/tools/gbench/Inputs/test3_run0.json new file mode 100644 index 0000000000..c777bb0ca9 --- /dev/null +++ b/tools/gbench/Inputs/test3_run0.json @@ -0,0 +1,39 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_One", + "iterations": 1000, + "real_time": 10, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_Two", + "iterations": 1000, + "real_time": 9, + "cpu_time": 90, + "time_unit": "ns" + }, + { + "name": "BM_Two", + "iterations": 1000, + "real_time": 8, + "cpu_time": 80, + "time_unit": "ns" + }, + { + "name": "BM_Two_stat", + "iterations": 1000, + "real_time": 8, + "cpu_time": 80, + "time_unit": "ns" + } + ] +} diff --git a/tools/gbench/Inputs/test3_run1.json b/tools/gbench/Inputs/test3_run1.json new file mode 100644 index 0000000000..035033391a --- /dev/null +++ b/tools/gbench/Inputs/test3_run1.json @@ -0,0 +1,39 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_One", + "iterations": 1000, + "real_time": 9, + "cpu_time": 110, + "time_unit": "ns" + }, + { + "name": "BM_Two", + "iterations": 1000, + "real_time": 10, + "cpu_time": 89, + "time_unit": "ns" + }, + { + "name": "BM_Two", + "iterations": 1000, + "real_time": 7, + "cpu_time": 70, + "time_unit": "ns" + }, + { + "name": "BM_Two_stat", + "iterations": 1000, + "real_time": 8, + "cpu_time": 80, + "time_unit": "ns" + } + ] +} diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 0c090981a8..4cdd3b74c1 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -4,6 +4,9 @@ import re import copy +from scipy.stats import mannwhitneyu + + class BenchmarkColor(object): def __init__(self, name, code): self.name = name @@ -16,11 +19,13 @@ def __repr__(self): def __format__(self, format): return self.code + # Benchmark Colors Enumeration BC_NONE = BenchmarkColor('NONE', '') BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m') BC_CYAN = BenchmarkColor('CYAN', '\033[96m') BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m') +BC_OKGREEN = BenchmarkColor('OKGREEN', '\033[32m') BC_HEADER = BenchmarkColor('HEADER', '\033[92m') BC_WARNING = BenchmarkColor('WARNING', '\033[93m') BC_WHITE = BenchmarkColor('WHITE', '\033[97m') @@ -29,6 +34,7 @@ def __format__(self, format): BC_BOLD = BenchmarkColor('BOLD', '\033[1m') BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m') + def color_format(use_color, fmt_str, *args, **kwargs): """ Return the result of 'fmt_str.format(*args, **kwargs)' after transforming @@ -78,30 +84,82 @@ def filter_benchmark(json_orig, family, replacement=""): for be in json_orig['benchmarks']: if not regex.search(be['name']): continue - filteredbench = copy.deepcopy(be) # Do NOT modify the old name! + filteredbench = copy.deepcopy(be) # Do NOT modify the old name! filteredbench['name'] = regex.sub(replacement, filteredbench['name']) filtered['benchmarks'].append(filteredbench) return filtered -def generate_difference_report(json1, json2, use_color=True): +def generate_difference_report( + json1, + json2, + utest=False, + utest_alpha=0.05, + use_color=True): """ Calculate and report the difference between each test of two benchmarks runs specified as 'json1' and 'json2'. """ + assert utest is True or utest is False first_col_width = find_longest_name(json1['benchmarks']) + def find_test(name): for b in json2['benchmarks']: if b['name'] == name: return b return None - first_col_width = max(first_col_width, len('Benchmark')) + + utest_col_name = "U-test (p-value)" + first_col_width = max( + first_col_width, + len('Benchmark'), + len(utest_col_name)) first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format( 'Benchmark', 12 + first_col_width) output_strs = [first_line, '-' * len(first_line)] - gen = (bn for bn in json1['benchmarks'] if 'real_time' in bn and 'cpu_time' in bn) + last_name = None + timings_time = [[], []] + timings_cpu = [[], []] + + gen = (bn for bn in json1['benchmarks'] + if 'real_time' in bn and 'cpu_time' in bn) for bn in gen: + fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" + special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}" + + if last_name is None: + last_name = bn['name'] + if last_name != bn['name']: + MIN_REPETITIONS = 2 + if ((len(timings_time[0]) >= MIN_REPETITIONS) and + (len(timings_time[1]) >= MIN_REPETITIONS) and + (len(timings_cpu[0]) >= MIN_REPETITIONS) and + (len(timings_cpu[1]) >= MIN_REPETITIONS)): + if utest: + def get_utest_color(pval): + if pval >= utest_alpha: + return BC_FAIL + else: + return BC_OKGREEN + time_pvalue = mannwhitneyu( + timings_time[0], timings_time[1], alternative='two-sided').pvalue + cpu_pvalue = mannwhitneyu( + timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue + output_strs += [color_format(use_color, + special_str, + BC_HEADER, + utest_col_name, + first_col_width, + get_utest_color(time_pvalue), + time_pvalue, + get_utest_color(cpu_pvalue), + cpu_pvalue, + endc=BC_ENDC)] + last_name = bn['name'] + timings_time = [[], []] + timings_cpu = [[], []] + other_bench = find_test(bn['name']) if not other_bench: continue @@ -116,26 +174,44 @@ def get_color(res): return BC_WHITE else: return BC_CYAN - fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" - tres = calculate_change(bn['real_time'], other_bench['real_time']) - cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time']) - output_strs += [color_format(use_color, fmt_str, - BC_HEADER, bn['name'], first_col_width, - get_color(tres), tres, get_color(cpures), cpures, - bn['real_time'], other_bench['real_time'], - bn['cpu_time'], other_bench['cpu_time'], - endc=BC_ENDC)] + + timings_time[0].append(bn['real_time']) + timings_time[1].append(other_bench['real_time']) + timings_cpu[0].append(bn['cpu_time']) + timings_cpu[1].append(other_bench['cpu_time']) + + tres = calculate_change(timings_time[0][-1], timings_time[1][-1]) + cpures = calculate_change(timings_cpu[0][-1], timings_cpu[1][-1]) + output_strs += [color_format(use_color, + fmt_str, + BC_HEADER, + bn['name'], + first_col_width, + get_color(tres), + tres, + get_color(cpures), + cpures, + timings_time[0][-1], + timings_time[1][-1], + timings_cpu[0][-1], + timings_cpu[1][-1], + endc=BC_ENDC)] return output_strs ############################################################################### # Unit tests + import unittest + class TestReportDifference(unittest.TestCase): def load_results(self): import json - testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs') + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') testOutput1 = os.path.join(testInputs, 'test1_run1.json') testOutput2 = os.path.join(testInputs, 'test1_run2.json') with open(testOutput1, 'r') as f: @@ -160,7 +236,8 @@ def test_basic(self): ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ] json1, json2 = self.load_results() - output_lines_with_header = generate_difference_report(json1, json2, use_color=False) + output_lines_with_header = generate_difference_report( + json1, json2, use_color=False) output_lines = output_lines_with_header[2:] print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) @@ -173,7 +250,10 @@ def test_basic(self): class TestReportDifferenceBetweenFamilies(unittest.TestCase): def load_result(self): import json - testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs') + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') testOutput = os.path.join(testInputs, 'test2_run.json') with open(testOutput, 'r') as f: json = json.load(f) @@ -189,7 +269,8 @@ def test_basic(self): json = self.load_result() json1 = filter_benchmark(json, "BM_Z.ro", ".") json2 = filter_benchmark(json, "BM_O.e", ".") - output_lines_with_header = generate_difference_report(json1, json2, use_color=False) + output_lines_with_header = generate_difference_report( + json1, json2, use_color=False) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) @@ -200,6 +281,41 @@ def test_basic(self): self.assertEqual(parts, expect_lines[i]) +class TestReportDifferenceWithUTest(unittest.TestCase): + def load_results(self): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput1 = os.path.join(testInputs, 'test3_run0.json') + testOutput2 = os.path.join(testInputs, 'test3_run1.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + def test_utest(self): + expect_lines = [] + expect_lines = [ + ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], + ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], + ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'], + ['U-test', '(p-value)', '0.2207', '0.6831'], + ['BM_Two_stat', '+0.0000', '+0.0000', '8', '8', '80', '80'], + ] + json1, json2 = self.load_results() + output_lines_with_header = generate_difference_report( + json1, json2, True, 0.05, use_color=False) + output_lines = output_lines_with_header[2:] + print("\n".join(output_lines_with_header)) + self.assertEqual(len(output_lines), len(expect_lines)) + for i in range(0, len(output_lines)): + parts = [x for x in output_lines[i].split(' ') if x] + self.assertEqual(parts, expect_lines[i]) + + if __name__ == '__main__': unittest.main() From c8adf4531f997c2701bec5084b5d92e37b632969 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 29 May 2018 13:12:48 +0100 Subject: [PATCH 046/330] Add some 'travis_wait' commands to avoid gcc@7 installation timeouts. (#605) --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09c058c544..168ed0b709 100644 --- a/.travis.yml +++ b/.travis.yml @@ -155,31 +155,31 @@ before_install: install: - if [ -n "${INSTALL_GCC6_FROM_PPA}" ]; then - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install g++-6; + travis_wait sudo -E apt-get -yq --no-install-suggests --no-install-recommends install g++-6; fi - if [ "${TRAVIS_OS_NAME}" == "linux" -a "${BUILD_32_BITS}" == "OFF" ]; then - sudo -E apt-get -y --no-install-suggests --no-install-recommends install llvm-3.9-tools; + travis_wait sudo -E apt-get -y --no-install-suggests --no-install-recommends install llvm-3.9-tools; sudo cp /usr/lib/llvm-3.9/bin/FileCheck /usr/local/bin/; fi - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then PATH=~/.local/bin:${PATH}; pip install --user --upgrade pip; - pip install --user cpp-coveralls; + travis_wait pip install --user cpp-coveralls; fi - if [ "${C_COMPILER}" == "gcc-7" -a "${TRAVIS_OS_NAME}" == "osx" ]; then rm -f /usr/local/include/c++; brew update; - brew install gcc@7; + travis_wait brew install gcc@7; fi - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo apt-get update -qq; sudo apt-get install -qq unzip; wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh; - sudo bash bazel-installer.sh; + travis_wait sudo bash bazel-installer.sh; fi - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-darwin-x86_64.sh; - sudo bash bazel-installer.sh; + travis_wait sudo bash bazel-installer.sh; fi script: From 16703ff83c1ae6d53e5155df3bb3ab0bc96083be Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 29 May 2018 13:13:06 +0100 Subject: [PATCH 047/330] cleaner and slightly larger statistics tests (#604) --- test/statistics_gtest.cc | 53 ++++++++-------------------------------- 1 file changed, 10 insertions(+), 43 deletions(-) diff --git a/test/statistics_gtest.cc b/test/statistics_gtest.cc index b4d6abbb57..e57460f75c 100644 --- a/test/statistics_gtest.cc +++ b/test/statistics_gtest.cc @@ -7,55 +7,22 @@ namespace { TEST(StatisticsTest, Mean) { - std::vector Inputs; - { - Inputs = {42, 42, 42, 42}; - double Res = benchmark::StatisticsMean(Inputs); - EXPECT_DOUBLE_EQ(Res, 42.0); - } - { - Inputs = {1, 2, 3, 4}; - double Res = benchmark::StatisticsMean(Inputs); - EXPECT_DOUBLE_EQ(Res, 2.5); - } - { - Inputs = {1, 2, 5, 10, 10, 14}; - double Res = benchmark::StatisticsMean(Inputs); - EXPECT_DOUBLE_EQ(Res, 7.0); - } + EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42,42,42,42}), 42.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1,2,3,4}), 2.5); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 5, 10, 10, 14}), 7.0); } TEST(StatisticsTest, Median) { - std::vector Inputs; - { - Inputs = {42, 42, 42, 42}; - double Res = benchmark::StatisticsMedian(Inputs); - EXPECT_DOUBLE_EQ(Res, 42.0); - } - { - Inputs = {1, 2, 3, 4}; - double Res = benchmark::StatisticsMedian(Inputs); - EXPECT_DOUBLE_EQ(Res, 2.5); - } - { - Inputs = {1, 2, 5, 10, 10}; - double Res = benchmark::StatisticsMedian(Inputs); - EXPECT_DOUBLE_EQ(Res, 5.0); - } + EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42,42,42,42}), 42.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1,2,3,4}), 2.5); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 5, 10, 10}), 5.0); } TEST(StatisticsTest, StdDev) { - std::vector Inputs; - { - Inputs = {101, 101, 101, 101}; - double Res = benchmark::StatisticsStdDev(Inputs); - EXPECT_DOUBLE_EQ(Res, 0.0); - } - { - Inputs = {1, 2, 3}; - double Res = benchmark::StatisticsStdDev(Inputs); - EXPECT_DOUBLE_EQ(Res, 1.0); - } + EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1,2,3}), 1.0); + EXPECT_FLOAT_EQ(benchmark::StatisticsStdDev({1.5, 2.4, 3.3, 4.2, 5.1}), + 1.42302495); } } // end namespace From 7b8d0249d8d66040ce9ffe6b94cc8421ef3e61d8 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 29 May 2018 06:25:32 -0600 Subject: [PATCH 048/330] Deprecate CSVReporter - A first step to overhauling reporting. (#488) As @dominichamon and I have discussed, the current reporter interface is poor at best. And something should be done to fix it. I strongly suspect such a fix will require an entire reimagining of the API, and therefore breaking backwards compatibility fully. For that reason we should start deprecating and removing parts that we don't intend to replace. One of these parts, I argue, is the CSVReporter. I propose that the new reporter interface should choose a single output format (JSON) and traffic entirely in that. If somebody really wanted to replace the functionality of the CSVReporter they would do so as an external tool which transforms the JSON. For these reasons I propose deprecating the CSVReporter. --- CMakeLists.txt | 4 +++- include/benchmark/benchmark.h | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1c1d3d5a9..f647c5999c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,6 @@ else() # Turn compiler warnings up to 11 add_cxx_compiler_flag(-Wall) - add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) add_cxx_compiler_flag(-Werror RELEASE) @@ -141,6 +140,9 @@ else() add_cxx_compiler_flag(-Wshorten-64-to-32) add_cxx_compiler_flag(-Wfloat-equal) add_cxx_compiler_flag(-fstrict-aliasing) + # Disable warnings regarding deprecated parts of the library while building + # and testing those parts of the library. + add_cxx_compiler_flag(-Wno-deprecated-declarations) if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 23dd3d09b1..f03b27ed40 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1414,7 +1414,8 @@ class JSONReporter : public BenchmarkReporter { bool first_report_; }; -class CSVReporter : public BenchmarkReporter { +class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future release") + CSVReporter : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} virtual bool ReportContext(const Context& context); From d07372e64ba16fe3d81bfffbdad7635d19a29198 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 29 May 2018 14:12:51 +0100 Subject: [PATCH 049/330] clang-format run on the benchmark header (#606) --- include/benchmark/benchmark.h | 132 ++++++++++++++++------------------ 1 file changed, 61 insertions(+), 71 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index f03b27ed40..58222508b3 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -164,7 +164,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #ifndef BENCHMARK_BENCHMARK_H_ #define BENCHMARK_BENCHMARK_H_ - // The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. #if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) #define BENCHMARK_HAS_CXX11 @@ -176,19 +175,19 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include #include -#include -#include #include #include +#include +#include #if defined(BENCHMARK_HAS_CXX11) -#include #include +#include #include #endif #if defined(_MSC_VER) -#include // for _ReadWriteBarrier +#include // for _ReadWriteBarrier #endif #ifndef BENCHMARK_HAS_CXX11 @@ -233,7 +232,9 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #else #define BENCHMARK_BUILTIN_EXPECT(x, y) x #define BENCHMARK_DEPRECATED_MSG(msg) -#define BENCHMARK_WARNING_MSG(msg) __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING(__LINE__) ") : warning note: " msg)) +#define BENCHMARK_WARNING_MSG(msg) \ + __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING( \ + __LINE__) ") : warning note: " msg)) #endif #if defined(__GNUC__) && !defined(__clang__) @@ -290,22 +291,19 @@ BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); } // namespace internal - #if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \ defined(__EMSCRIPTEN__) -# define BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#define BENCHMARK_HAS_NO_INLINE_ASSEMBLY #endif - // The DoNotOptimize(...) function can be used to prevent a value or // expression from being optimized away by the compiler. This function is // intended to add little to no overhead. // See: https://youtu.be/nXaxk27zwlk?t=2441 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY template -inline BENCHMARK_ALWAYS_INLINE -void DoNotOptimize(Tp const& value) { - asm volatile("" : : "r,m"(value) : "memory"); +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); } template @@ -329,9 +327,7 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { _ReadWriteBarrier(); } -inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { - _ReadWriteBarrier(); -} +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } #else template inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { @@ -340,39 +336,34 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers #endif - - // This class is used for user-defined counters. class Counter { -public: - + public: enum Flags { - kDefaults = 0, + kDefaults = 0, // Mark the counter as a rate. It will be presented divided // by the duration of the benchmark. - kIsRate = 1, + kIsRate = 1, // Mark the counter as a thread-average quantity. It will be // presented divided by the number of threads. kAvgThreads = 2, // Mark the counter as a thread-average rate. See above. - kAvgThreadsRate = kIsRate|kAvgThreads + kAvgThreadsRate = kIsRate | kAvgThreads }; double value; - Flags flags; + Flags flags; BENCHMARK_ALWAYS_INLINE Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {} - BENCHMARK_ALWAYS_INLINE operator double const& () const { return value; } - BENCHMARK_ALWAYS_INLINE operator double & () { return value; } - + BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; } + BENCHMARK_ALWAYS_INLINE operator double&() { return value; } }; // This is the container for the user-defined counters. typedef std::map UserCounters; - // TimeUnit is passed to a benchmark in order to specify the order of magnitude // for the measured time. enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; @@ -396,7 +387,7 @@ struct Statistics { StatisticsFunc* compute_; Statistics(std::string name, StatisticsFunc* compute) - : name_(name), compute_(compute) {} + : name_(name), compute_(compute) {} }; namespace internal { @@ -405,14 +396,12 @@ class ThreadManager; enum ReportMode #if defined(BENCHMARK_HAS_CXX11) - : unsigned + : unsigned #else #endif - { - RM_Unspecified, // The mode has not been manually specified +{ RM_Unspecified, // The mode has not been manually specified RM_Default, // The mode is user-specified as default. - RM_ReportAggregatesOnly -}; + RM_ReportAggregatesOnly }; } // namespace internal // State is passed to a running Benchmark and contains state for the @@ -581,8 +570,8 @@ class State { return max_iterations - total_iterations_ + batch_leftover_; } -private: // items we expect on the first cache line (ie 64 bytes of the struct) - + private + : // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. // May be larger than max_iterations. size_t total_iterations_; @@ -592,15 +581,15 @@ class State { // completed_iterations_ accurately. size_t batch_leftover_; -public: + public: const size_t max_iterations; -private: + private: bool started_; bool finished_; bool error_occurred_; -private: // items we don't need on the first cache line + private: // items we don't need on the first cache line std::vector range_; int64_t bytes_processed_; @@ -616,7 +605,6 @@ class State { // Number of threads concurrently executing the benchmark. const int threads; - // TODO(EricWF) make me private State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, @@ -633,18 +621,16 @@ class State { BENCHMARK_DISALLOW_COPY_AND_ASSIGN(State); }; -inline BENCHMARK_ALWAYS_INLINE -bool State::KeepRunning() { - return KeepRunningInternal(1, /*is_batch=*/ false); +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { + return KeepRunningInternal(1, /*is_batch=*/false); } -inline BENCHMARK_ALWAYS_INLINE -bool State::KeepRunningBatch(size_t n) { - return KeepRunningInternal(n, /*is_batch=*/ true); +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(size_t n) { + return KeepRunningInternal(n, /*is_batch=*/true); } -inline BENCHMARK_ALWAYS_INLINE -bool State::KeepRunningInternal(size_t n, bool is_batch) { +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(size_t n, + bool is_batch) { // total_iterations_ is set to 0 by the constructor, and always set to a // nonzero value by StartKepRunning(). assert(n > 0); @@ -657,13 +643,13 @@ bool State::KeepRunningInternal(size_t n, bool is_batch) { if (!started_) { StartKeepRunning(); if (!error_occurred_ && total_iterations_ >= n) { - total_iterations_-= n; + total_iterations_ -= n; return true; } } // For non-batch runs, total_iterations_ must be 0 by now. if (is_batch && total_iterations_ != 0) { - batch_leftover_ = n - total_iterations_; + batch_leftover_ = n - total_iterations_; total_iterations_ = 0; return true; } @@ -897,7 +883,7 @@ class Benchmark { std::string name_; ReportMode report_mode_; - std::vector arg_names_; // Args for all benchmark runs + std::vector arg_names_; // Args for all benchmark runs std::vector > args_; // Args for all benchmark runs TimeUnit time_unit_; int range_multiplier_; @@ -1122,7 +1108,7 @@ class Fixture : public internal::Benchmark { class BaseClass##_##Method##_Benchmark : public BaseClass { \ public: \ BaseClass##_##Method##_Benchmark() : BaseClass() { \ - this->SetName(#BaseClass"<" #a ">/" #Method); \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ } \ \ protected: \ @@ -1133,7 +1119,7 @@ class Fixture : public internal::Benchmark { class BaseClass##_##Method##_Benchmark : public BaseClass { \ public: \ BaseClass##_##Method##_Benchmark() : BaseClass() { \ - this->SetName(#BaseClass"<" #a "," #b ">/" #Method); \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ } \ \ protected: \ @@ -1145,14 +1131,15 @@ class Fixture : public internal::Benchmark { class BaseClass##_##Method##_Benchmark : public BaseClass<__VA_ARGS__> { \ public: \ BaseClass##_##Method##_Benchmark() : BaseClass<__VA_ARGS__>() { \ - this->SetName(#BaseClass"<" #__VA_ARGS__ ">/" #Method); \ + this->SetName(#BaseClass "<" #__VA_ARGS__ ">/" #Method); \ } \ \ protected: \ virtual void BenchmarkCase(::benchmark::State&); \ }; #else -#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) +#define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ + BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(n, a) #endif #define BENCHMARK_DEFINE_F(BaseClass, Method) \ @@ -1172,7 +1159,8 @@ class Fixture : public internal::Benchmark { BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ void BaseClass##_##Method##_Benchmark::BenchmarkCase #else -#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) +#define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) #endif #define BENCHMARK_REGISTER_F(BaseClass, Method) \ @@ -1199,24 +1187,24 @@ class Fixture : public internal::Benchmark { void BaseClass##_##Method##_Benchmark::BenchmarkCase #ifdef BENCHMARK_HAS_CXX11 -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ +#define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - BENCHMARK_REGISTER_F(BaseClass, Method); \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ void BaseClass##_##Method##_Benchmark::BenchmarkCase #else -#define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) +#define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) \ + BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) #endif // Helper macro to create a main routine in a test that runs the benchmarks -#define BENCHMARK_MAIN() \ - int main(int argc, char** argv) { \ - ::benchmark::Initialize(&argc, argv); \ +#define BENCHMARK_MAIN() \ + int main(int argc, char** argv) { \ + ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ - ::benchmark::RunSpecifiedBenchmarks(); \ - } \ + ::benchmark::RunSpecifiedBenchmarks(); \ + } \ int main(int, char**) - // ------------------------------------------------------ // Benchmark Reporters @@ -1253,7 +1241,7 @@ class BenchmarkReporter { CPUInfo const& cpu_info; // The number of chars in the longest benchmark name. size_t name_field_width; - static const char *executable_name; + static const char* executable_name; Context(); }; @@ -1376,17 +1364,19 @@ class BenchmarkReporter { // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { -public: + public: enum OutputOptions { OO_None = 0, OO_Color = 1, OO_Tabular = 2, - OO_ColorTabular = OO_Color|OO_Tabular, + OO_ColorTabular = OO_Color | OO_Tabular, OO_Defaults = OO_ColorTabular }; explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) - : output_options_(opts_), name_field_width_(0), - prev_counters_(), printed_header_(false) {} + : output_options_(opts_), + name_field_width_(0), + prev_counters_(), + printed_header_(false) {} virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); @@ -1425,7 +1415,7 @@ class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future rel void PrintRunData(const Run& report); bool printed_header_; - std::set< std::string > user_counter_names_; + std::set user_counter_names_; }; inline const char* GetTimeUnitString(TimeUnit unit) { @@ -1452,6 +1442,6 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { } } -} // namespace benchmark +} // namespace benchmark #endif // BENCHMARK_BENCHMARK_H_ From 4fbfa2f3368cb8d8a0cba48edda584c7dd9f0a14 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 30 May 2018 13:17:41 +0100 Subject: [PATCH 050/330] Some platforms and environments don't pass a valid argc/argv. (#607) Specifically some iOS targets. --- src/benchmark.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 82b15ac709..8bf87d11ad 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -568,7 +568,8 @@ void PrintUsageAndExit() { void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; - BenchmarkReporter::Context::executable_name = argv[0]; + BenchmarkReporter::Context::executable_name = + (argc && *argc > 0) ? argv[0] : "unknown"; for (int i = 1; i < *argc; ++i) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || From 4c2af0788977d3bd900585528c2d08b875b2cd39 Mon Sep 17 00:00:00 2001 From: BaaMeow <38274252+BaaMeow@users.noreply.github.com> Date: Fri, 1 Jun 2018 06:14:19 -0400 Subject: [PATCH 051/330] (clang-)format all the things (#610) * format all documents according to contributor guidelines and specifications use clang-format on/off to stop formatting when it makes excessively poor decisions * format all tests as well, and mark blocks which change too much --- src/benchmark.cc | 28 ++++++---- src/benchmark_register.cc | 6 +- src/check.h | 5 +- src/commandlineflags.cc | 2 +- src/counter.cc | 14 ++--- src/counter.h | 8 +-- src/csv_reporter.cc | 20 +++---- src/cycleclock.h | 6 +- src/internal_macros.h | 4 ++ src/json_reporter.cc | 36 +++++------- src/log.h | 3 +- src/re.h | 24 +++++--- src/reporter.cc | 2 +- src/statistics.cc | 25 ++++----- src/string_util.h | 3 +- test/output_test.h | 27 +++++---- test/output_test_helper.cc | 88 +++++++++++++++--------------- test/register_benchmark_test.cc | 2 + test/reporter_output_test.cc | 31 ++++++----- test/skip_with_error_test.cc | 9 +-- test/state_assembly_test.cc | 2 + test/statistics_gtest.cc | 12 ++-- test/templated_fixture_test.cc | 6 +- test/user_counters_tabular_test.cc | 66 +++++++++++----------- test/user_counters_test.cc | 72 +++++++++++++++--------- 25 files changed, 266 insertions(+), 235 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 8bf87d11ad..9422cdb164 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -115,8 +115,7 @@ namespace { BenchmarkReporter::Run CreateRunReport( const benchmark::internal::Benchmark::Instance& b, - const internal::ThreadManager::Result& results, - double seconds) { + const internal::ThreadManager::Result& results, double seconds) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -234,6 +233,8 @@ std::vector RunBenchmark( const double min_time = !IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time; + // clang-format off + // turn off clang-format since it mangles prettiness here // Determine if this run should be reported; Either it has // run for a sufficient amount of time or because an error was reported. const bool should_report = repetition_num > 0 @@ -245,6 +246,7 @@ std::vector RunBenchmark( // minimum time. Note that user provided timers are except from this // sanity check. || ((results.real_time_used >= 5 * min_time) && !b.use_manual_time); + // clang-format on if (should_report) { BenchmarkReporter::Run report = CreateRunReport(b, results, seconds); @@ -324,7 +326,8 @@ State::State(size_t max_iters, const std::vector& ranges, int thread_i, // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; static_assert(offsetof(State, error_occurred_) <= - (cache_line_size - sizeof(error_occurred_)), ""); + (cache_line_size - sizeof(error_occurred_)), + ""); #ifdef __GNUC__ #pragma GCC diagnostic pop #endif @@ -387,8 +390,8 @@ namespace internal { namespace { void RunBenchmarks(const std::vector& benchmarks, - BenchmarkReporter* console_reporter, - BenchmarkReporter* file_reporter) { + BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter) { // Note the file_reporter can be null. CHECK(console_reporter != nullptr); @@ -401,7 +404,7 @@ void RunBenchmarks(const std::vector& benchmarks, std::max(name_field_width, benchmark.name.size()); has_repetitions |= benchmark.repetitions > 1; - for(const auto& Stat : *benchmark.statistics) + for (const auto& Stat : *benchmark.statistics) stat_field_width = std::max(stat_field_width, Stat.name_.size()); } if (has_repetitions) name_field_width += 1 + stat_field_width; @@ -469,15 +472,15 @@ ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { } else { output_opts &= ~ConsoleReporter::OO_Color; } - if(force_no_color) { + if (force_no_color) { output_opts &= ~ConsoleReporter::OO_Color; } - if(FLAGS_benchmark_counters_tabular) { + if (FLAGS_benchmark_counters_tabular) { output_opts |= ConsoleReporter::OO_Tabular; } else { output_opts &= ~ConsoleReporter::OO_Tabular; } - return static_cast< ConsoleReporter::OutputOptions >(output_opts); + return static_cast(output_opts); } } // end namespace internal @@ -502,7 +505,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, std::unique_ptr default_file_reporter; if (!console_reporter) { default_console_reporter = internal::CreateReporter( - FLAGS_benchmark_format, internal::GetOutputOptions()); + FLAGS_benchmark_format, internal::GetOutputOptions()); console_reporter = default_console_reporter.get(); } auto& Out = console_reporter->GetOutputStream(); @@ -589,7 +592,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { // TODO: Remove this. ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || ParseBoolFlag(argv[i], "benchmark_counters_tabular", - &FLAGS_benchmark_counters_tabular) || + &FLAGS_benchmark_counters_tabular) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; @@ -623,7 +626,8 @@ void Initialize(int* argc, char** argv) { bool ReportUnrecognizedArguments(int argc, char** argv) { for (int i = 1; i < argc; ++i) { - fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], argv[i]); + fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], + argv[i]); } return argc > 1; } diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index dc6f935685..26a89721c7 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -115,9 +115,9 @@ bool BenchmarkFamilies::FindBenchmarks( std::string error_msg; Regex re; bool isNegativeFilter = false; - if(spec[0] == '-') { - spec.replace(0, 1, ""); - isNegativeFilter = true; + if (spec[0] == '-') { + spec.replace(0, 1, ""); + isNegativeFilter = true; } if (!re.Init(spec, &error_msg)) { Err << "Could not compile benchmark re: " << error_msg << std::endl; diff --git a/src/check.h b/src/check.h index 73bead2fb5..f5f8253f80 100644 --- a/src/check.h +++ b/src/check.h @@ -1,9 +1,9 @@ #ifndef CHECK_H_ #define CHECK_H_ +#include #include #include -#include #include "internal_macros.h" #include "log.h" @@ -62,6 +62,8 @@ class CheckHandler { #define CHECK(b) ::benchmark::internal::GetNullLogInstance() #endif +// clang-format off +// preserve whitespacing between operators for alignment #define CHECK_EQ(a, b) CHECK((a) == (b)) #define CHECK_NE(a, b) CHECK((a) != (b)) #define CHECK_GE(a, b) CHECK((a) >= (b)) @@ -75,5 +77,6 @@ class CheckHandler { #define CHECK_FLOAT_LE(a, b, eps) CHECK((b) - (a) > -(eps)) #define CHECK_FLOAT_GT(a, b, eps) CHECK((a) - (b) > (eps)) #define CHECK_FLOAT_LT(a, b, eps) CHECK((b) - (a) > (eps)) +//clang-format on #endif // CHECK_H_ diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 2fc92517a3..734e88bbec 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -45,7 +45,7 @@ bool ParseInt32(const std::string& src_text, const char* str, int32_t* value) { // LONG_MAX or LONG_MIN when the input overflows.) result != long_value // The parsed value overflows as an Int32. - ) { + ) { std::cerr << src_text << " is expected to be a 32-bit integer, " << "but actually has value \"" << str << "\", " << "which overflows.\n"; diff --git a/src/counter.cc b/src/counter.cc index ed1aa044ee..50d89a6a7a 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -28,22 +28,22 @@ double Finish(Counter const& c, double cpu_time, double num_threads) { return v; } -void Finish(UserCounters *l, double cpu_time, double num_threads) { - for (auto &c : *l) { +void Finish(UserCounters* l, double cpu_time, double num_threads) { + for (auto& c : *l) { c.second.value = Finish(c.second, cpu_time, num_threads); } } -void Increment(UserCounters *l, UserCounters const& r) { +void Increment(UserCounters* l, UserCounters const& r) { // add counters present in both or just in *l - for (auto &c : *l) { + for (auto& c : *l) { auto it = r.find(c.first); if (it != r.end()) { c.second.value = c.second + it->second; } } // add counters present in r, but not in *l - for (auto const &tc : r) { + for (auto const& tc : r) { auto it = l->find(tc.first); if (it == l->end()) { (*l)[tc.first] = tc.second; @@ -64,5 +64,5 @@ bool SameNames(UserCounters const& l, UserCounters const& r) { return true; } -} // end namespace internal -} // end namespace benchmark +} // end namespace internal +} // end namespace benchmark diff --git a/src/counter.h b/src/counter.h index dd6865a31d..299cc0266c 100644 --- a/src/counter.h +++ b/src/counter.h @@ -18,9 +18,9 @@ namespace benchmark { // these counter-related functions are hidden to reduce API surface. namespace internal { -void Finish(UserCounters *l, double time, double num_threads); -void Increment(UserCounters *l, UserCounters const& r); +void Finish(UserCounters* l, double time, double num_threads); +void Increment(UserCounters* l, UserCounters const& r); bool SameNames(UserCounters const& l, UserCounters const& r); -} // end namespace internal +} // end namespace internal -} //end namespace benchmark +} // end namespace benchmark diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 35510645b0..4a641909d8 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -22,9 +22,9 @@ #include #include +#include "check.h" #include "string_util.h" #include "timers.h" -#include "check.h" // File format reference: http://edoceo.com/utilitas/csv-file-format. @@ -42,7 +42,7 @@ bool CSVReporter::ReportContext(const Context& context) { return true; } -void CSVReporter::ReportRuns(const std::vector & reports) { +void CSVReporter::ReportRuns(const std::vector& reports) { std::ostream& Out = GetOutputStream(); if (!printed_header_) { @@ -58,7 +58,8 @@ void CSVReporter::ReportRuns(const std::vector & reports) { Out << *B++; if (B != elements.end()) Out << ","; } - for (auto B = user_counter_names_.begin(); B != user_counter_names_.end();) { + for (auto B = user_counter_names_.begin(); + B != user_counter_names_.end();) { Out << ",\"" << *B++ << "\""; } Out << "\n"; @@ -69,9 +70,9 @@ void CSVReporter::ReportRuns(const std::vector & reports) { for (const auto& run : reports) { for (const auto& cnt : run.counters) { CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) - << "All counters must be present in each run. " - << "Counter named \"" << cnt.first - << "\" was not in a run after being added to the header"; + << "All counters must be present in each run. " + << "Counter named \"" << cnt.first + << "\" was not in a run after being added to the header"; } } } @@ -80,10 +81,9 @@ void CSVReporter::ReportRuns(const std::vector & reports) { for (const auto& run : reports) { PrintRunData(run); } - } -void CSVReporter::PrintRunData(const Run & run) { +void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); // Field with embedded double-quote characters must be doubled and the field @@ -135,9 +135,9 @@ void CSVReporter::PrintRunData(const Run & run) { Out << ",,"; // for error_occurred and error_message // Print user counters - for (const auto &ucn : user_counter_names_) { + for (const auto& ucn : user_counter_names_) { auto it = run.counters.find(ucn); - if(it == run.counters.end()) { + if (it == run.counters.end()) { Out << ","; } else { Out << "," << it->second; diff --git a/src/cycleclock.h b/src/cycleclock.h index 3b376ac57d..00d5764167 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -121,7 +121,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // because is provides nanosecond resolution (which is noticable at // least for PNaCl modules running on x86 Mac & Linux). // Initialize to always return 0 if clock_gettime fails. - struct timespec ts = { 0, 0 }; + struct timespec ts = {0, 0}; clock_gettime(CLOCK_MONOTONIC, &ts); return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; #elif defined(__aarch64__) @@ -159,10 +159,10 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__s390__) // Covers both s390 and s390x. +#elif defined(__s390__) // Covers both s390 and s390x. // Return the CPU clock. uint64_t tsc; - asm("stck %0" : "=Q" (tsc) : : "cc"); + asm("stck %0" : "=Q"(tsc) : : "cc"); return tsc; #else // The soft failover to a generic implementation is automatic only for ARM. diff --git a/src/internal_macros.h b/src/internal_macros.h index edb8a5c0a3..500f0ababc 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -3,6 +3,8 @@ #include "benchmark/benchmark.h" +// clang-format off + #ifndef __has_feature #define __has_feature(x) 0 #endif @@ -86,4 +88,6 @@ #define BENCHMARK_UNREACHABLE() ((void)0) #endif +// clang-format on + #endif // BENCHMARK_INTERNAL_MACROS_H_ diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 685d6b097d..611605af6b 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -17,12 +17,12 @@ #include #include +#include // for setprecision #include +#include #include #include #include -#include // for setprecision -#include #include "string_util.h" #include "timers.h" @@ -53,7 +53,7 @@ std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << key << "\": "; - const auto max_digits10 = std::numeric_limits::max_digits10; + const auto max_digits10 = std::numeric_limits::max_digits10; const auto max_fractional_digits10 = max_digits10 - 1; ss << std::scientific << std::setprecision(max_fractional_digits10) << value; @@ -161,40 +161,30 @@ void JSONReporter::PrintRunData(Run const& run) { } if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; - out << indent - << FormatKV("real_time", run.GetAdjustedRealTime()) - << ",\n"; - out << indent - << FormatKV("cpu_time", run.GetAdjustedCPUTime()); + out << indent << FormatKV("real_time", run.GetAdjustedRealTime()) << ",\n"; + out << indent << FormatKV("cpu_time", run.GetAdjustedCPUTime()); out << ",\n" << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); } else if (run.report_big_o) { - out << indent - << FormatKV("cpu_coefficient", run.GetAdjustedCPUTime()) + out << indent << FormatKV("cpu_coefficient", run.GetAdjustedCPUTime()) << ",\n"; - out << indent - << FormatKV("real_coefficient", run.GetAdjustedRealTime()) + out << indent << FormatKV("real_coefficient", run.GetAdjustedRealTime()) << ",\n"; out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n"; out << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); } else if (run.report_rms) { - out << indent - << FormatKV("rms", run.GetAdjustedCPUTime()); + out << indent << FormatKV("rms", run.GetAdjustedCPUTime()); } if (run.bytes_per_second > 0.0) { out << ",\n" - << indent - << FormatKV("bytes_per_second", run.bytes_per_second); + << indent << FormatKV("bytes_per_second", run.bytes_per_second); } if (run.items_per_second > 0.0) { out << ",\n" - << indent - << FormatKV("items_per_second", run.items_per_second); + << indent << FormatKV("items_per_second", run.items_per_second); } - for(auto &c : run.counters) { - out << ",\n" - << indent - << FormatKV(c.first, c.second); + for (auto& c : run.counters) { + out << ",\n" << indent << FormatKV(c.first, c.second); } if (!run.report_label.empty()) { out << ",\n" << indent << FormatKV("label", run.report_label); @@ -202,4 +192,4 @@ void JSONReporter::PrintRunData(Run const& run) { out << '\n'; } -} // end namespace benchmark +} // end namespace benchmark diff --git a/src/log.h b/src/log.h index d06e1031db..47d0c35c01 100644 --- a/src/log.h +++ b/src/log.h @@ -66,8 +66,9 @@ inline LogType& GetLogInstanceForLevel(int level) { } // end namespace internal } // end namespace benchmark +// clang-format off #define VLOG(x) \ (::benchmark::internal::GetLogInstanceForLevel(x) << "-- LOG(" << x << "):" \ " ") - +// clang-format on #endif diff --git a/src/re.h b/src/re.h index 924d2f0ba7..fbe25037b4 100644 --- a/src/re.h +++ b/src/re.h @@ -17,6 +17,8 @@ #include "internal_macros.h" +// clang-format off + #if !defined(HAVE_STD_REGEX) && \ !defined(HAVE_GNU_POSIX_REGEX) && \ !defined(HAVE_POSIX_REGEX) @@ -45,6 +47,9 @@ #else #error No regular expression backend was found! #endif + +// clang-format on + #include #include "check.h" @@ -76,7 +81,7 @@ class Regex { #elif defined(HAVE_POSIX_REGEX) || defined(HAVE_GNU_POSIX_REGEX) regex_t re_; #else - #error No regular expression backend implementation available +#error No regular expression backend implementation available #endif }; @@ -84,20 +89,21 @@ class Regex { inline bool Regex::Init(const std::string& spec, std::string* error) { #ifdef BENCHMARK_HAS_NO_EXCEPTIONS - ((void)error); // suppress unused warning + ((void)error); // suppress unused warning #else try { #endif - re_ = std::regex(spec, std::regex_constants::extended); - init_ = true; + re_ = std::regex(spec, std::regex_constants::extended); + init_ = true; #ifndef BENCHMARK_HAS_NO_EXCEPTIONS - } catch (const std::regex_error& e) { - if (error) { - *error = e.what(); - } +} +catch (const std::regex_error& e) { + if (error) { + *error = e.what(); } +} #endif - return init_; +return init_; } inline Regex::~Regex() {} diff --git a/src/reporter.cc b/src/reporter.cc index 4b40aaec8b..541661a25f 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -68,7 +68,7 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, } // No initializer because it's already initialized to NULL. -const char* BenchmarkReporter::Context::executable_name; +const char *BenchmarkReporter::Context::executable_name; BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()) {} diff --git a/src/statistics.cc b/src/statistics.cc index 1c91e1015a..612dda2d1a 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -17,9 +17,9 @@ #include #include +#include #include #include -#include #include "check.h" #include "statistics.h" @@ -43,9 +43,9 @@ double StatisticsMedian(const std::vector& v) { // did we have an odd number of samples? // if yes, then center is the median - // it no, then we are looking for the average between center and the value before - if(v.size() % 2 == 1) - return *center; + // it no, then we are looking for the average between center and the value + // before + if (v.size() % 2 == 1) return *center; auto center2 = copy.begin() + v.size() / 2 - 1; std::nth_element(copy.begin(), center2, copy.end()); return (*center + *center2) / 2.0; @@ -68,8 +68,7 @@ double StatisticsStdDev(const std::vector& v) { if (v.empty()) return mean; // Sample standard deviation is undefined for n = 1 - if (v.size() == 1) - return 0.0; + if (v.size() == 1) return 0.0; const double avg_squares = SumSquares(v) * (1.0 / v.size()); return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); @@ -108,11 +107,11 @@ std::vector ComputeStats( Counter c; std::vector s; }; - std::map< std::string, CounterStat > counter_stats; - for(Run const& r : reports) { - for(auto const& cnt : r.counters) { + std::map counter_stats; + for (Run const& r : reports) { + for (auto const& cnt : r.counters) { auto it = counter_stats.find(cnt.first); - if(it == counter_stats.end()) { + if (it == counter_stats.end()) { counter_stats.insert({cnt.first, {cnt.second, std::vector{}}}); it = counter_stats.find(cnt.first); it->second.s.reserve(reports.size()); @@ -132,7 +131,7 @@ std::vector ComputeStats( items_per_second_stat.emplace_back(run.items_per_second); bytes_per_second_stat.emplace_back(run.bytes_per_second); // user counters - for(auto const& cnt : run.counters) { + for (auto const& cnt : run.counters) { auto it = counter_stats.find(cnt.first); CHECK_NE(it, counter_stats.end()); it->second.s.emplace_back(cnt.second); @@ -148,7 +147,7 @@ std::vector ComputeStats( } } - for(const auto& Stat : *reports[0].statistics) { + for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_; @@ -163,7 +162,7 @@ std::vector ComputeStats( data.time_unit = reports[0].time_unit; // user counters - for(auto const& kv : counter_stats) { + for (auto const& kv : counter_stats) { const auto uc_stat = Stat.compute_(kv.second.s); auto c = Counter(uc_stat, counter_stats[kv.first].c.flags); data.counters[kv.first] = c; diff --git a/src/string_util.h b/src/string_util.h index e70e769872..ca49c4fde1 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -19,8 +19,7 @@ inline std::ostream& StrCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { } template -inline std::ostream& StrCatImp(std::ostream& out, First&& f, - Rest&&... rest) { +inline std::ostream& StrCatImp(std::ostream& out, First&& f, Rest&&... rest) { out << std::forward(f); return StrCatImp(out, std::forward(rest)...); } diff --git a/test/output_test.h b/test/output_test.h index 897a13866b..b157fe66a2 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -2,13 +2,13 @@ #define TEST_OUTPUT_TEST_H #undef NDEBUG +#include #include #include +#include #include #include #include -#include -#include #include "../src/re.h" #include "benchmark/benchmark.h" @@ -73,21 +73,20 @@ void RunOutputTests(int argc, char* argv[]); // will be the subject of a call to checker_function // checker_function: should be of type ResultsCheckFn (see below) #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ - size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) + size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) struct Results; -typedef std::function< void(Results const&) > ResultsCheckFn; +typedef std::function ResultsCheckFn; size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn); // Class holding the results of a benchmark. // It is passed in calls to checker functions. struct Results { - // the benchmark name std::string name; // the benchmark fields - std::map< std::string, std::string > values; + std::map values; Results(const std::string& n) : name(n) {} @@ -102,18 +101,18 @@ struct Results { // it is better to use fuzzy float checks for this, as the float // ASCII formatting is lossy. double DurationRealTime() const { - return GetAs< double >("iterations") * GetTime(kRealTime); + return GetAs("iterations") * GetTime(kRealTime); } // get the cpu_time duration of the benchmark in seconds double DurationCPUTime() const { - return GetAs< double >("iterations") * GetTime(kCpuTime); + return GetAs("iterations") * GetTime(kCpuTime); } // get the string for a result by name, or nullptr if the name // is not found const std::string* Get(const char* entry_name) const { auto it = values.find(entry_name); - if(it == values.end()) return nullptr; + if (it == values.end()) return nullptr; return &it->second; } @@ -126,15 +125,15 @@ struct Results { // as a double, and only then converted to the asked type. template T GetCounterAs(const char* entry_name) const { - double dval = GetAs< double >(entry_name); - T tval = static_cast< T >(dval); + double dval = GetAs(entry_name); + T tval = static_cast(dval); return tval; } }; template T Results::GetAs(const char* entry_name) const { - auto *sv = Get(entry_name); + auto* sv = Get(entry_name); CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; ss << *sv; @@ -148,6 +147,8 @@ T Results::GetAs(const char* entry_name) const { // Macros to help in result checking. Do not use them with arguments causing // side-effects. +// clang-format off + #define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ CONCAT(CHECK_, relationship) \ (entry.getfn< var_type >(var_name), (value)) << "\n" \ @@ -188,6 +189,8 @@ T Results::GetAs(const char* entry_name) const { #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) +// clang-format on + // ========================================================================= // // --------------------------- Misc Utilities ------------------------------ // // ========================================================================= // diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 6b18fe4359..92c2448d9e 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -1,13 +1,13 @@ +#include #include #include #include #include -#include +#include "../src/benchmark_api_internal.h" #include "../src/check.h" // NOTE: check.h is for internal use only! #include "../src/re.h" // NOTE: re.h is for internal use only #include "output_test.h" -#include "../src/benchmark_api_internal.h" // ========================================================================= // // ------------------------------ Internals -------------------------------- // @@ -33,6 +33,7 @@ TestCaseList& GetTestCaseList(TestCaseID ID) { SubMap& GetSubstitutions() { // Don't use 'dec_re' from header because it may not yet be initialized. + // clang-format off static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, @@ -57,6 +58,7 @@ SubMap& GetSubstitutions() { "," + safe_dec_re + ",,,"}, {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, {"%csv_label_report_end", ",,"}}; + // clang-format on return map; } @@ -147,9 +149,9 @@ class TestReporter : public benchmark::BenchmarkReporter { } private: - std::vector reporters_; + std::vector reporters_; }; -} +} // namespace } // end namespace internal @@ -163,28 +165,25 @@ namespace internal { // It works by parsing the CSV output to read the results. class ResultsChecker { public: - - struct PatternAndFn : public TestCase { // reusing TestCase for its regexes + struct PatternAndFn : public TestCase { // reusing TestCase for its regexes PatternAndFn(const std::string& rx, ResultsCheckFn fn_) - : TestCase(rx), fn(fn_) {} + : TestCase(rx), fn(fn_) {} ResultsCheckFn fn; }; - std::vector< PatternAndFn > check_patterns; - std::vector< Results > results; - std::vector< std::string > field_names; + std::vector check_patterns; + std::vector results; + std::vector field_names; void Add(const std::string& entry_pattern, ResultsCheckFn fn); void CheckResults(std::stringstream& output); private: - void SetHeader_(const std::string& csv_header); void SetValues_(const std::string& entry_csv_line); - std::vector< std::string > SplitCsv_(const std::string& line); - + std::vector SplitCsv_(const std::string& line); }; // store the static ResultsChecker in a function to prevent initialization @@ -207,7 +206,7 @@ void ResultsChecker::CheckResults(std::stringstream& output) { // clear before calling tellg() output.clear(); // seek to zero only when needed - if(output.tellg() > start) output.seekg(start); + if (output.tellg() > start) output.seekg(start); // and just in case output.clear(); } @@ -218,18 +217,18 @@ void ResultsChecker::CheckResults(std::stringstream& output) { CHECK(output.good()); std::getline(output, line); if (on_first) { - SetHeader_(line); // this is important + SetHeader_(line); // this is important on_first = false; continue; } SetValues_(line); } // finally we can call the subscribed check functions - for(const auto& p : check_patterns) { + for (const auto& p : check_patterns) { VLOG(2) << "--------------------------------\n"; VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; - for(const auto& r : results) { - if(!p.regex->Match(r.name)) { + for (const auto& r : results) { + if (!p.regex->Match(r.name)) { VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; } else { @@ -249,51 +248,50 @@ void ResultsChecker::SetHeader_(const std::string& csv_header) { // set the values for a benchmark void ResultsChecker::SetValues_(const std::string& entry_csv_line) { - if(entry_csv_line.empty()) return; // some lines are empty + if (entry_csv_line.empty()) return; // some lines are empty CHECK(!field_names.empty()); auto vals = SplitCsv_(entry_csv_line); CHECK_EQ(vals.size(), field_names.size()); - results.emplace_back(vals[0]); // vals[0] is the benchmark name - auto &entry = results.back(); + results.emplace_back(vals[0]); // vals[0] is the benchmark name + auto& entry = results.back(); for (size_t i = 1, e = vals.size(); i < e; ++i) { entry.values[field_names[i]] = vals[i]; } } // a quick'n'dirty csv splitter (eliminating quotes) -std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) { - std::vector< std::string > out; - if(line.empty()) return out; - if(!field_names.empty()) out.reserve(field_names.size()); +std::vector ResultsChecker::SplitCsv_(const std::string& line) { + std::vector out; + if (line.empty()) return out; + if (!field_names.empty()) out.reserve(field_names.size()); size_t prev = 0, pos = line.find_first_of(','), curr = pos; - while(pos != line.npos) { + while (pos != line.npos) { CHECK(curr > 0); - if(line[prev] == '"') ++prev; - if(line[curr-1] == '"') --curr; - out.push_back(line.substr(prev, curr-prev)); + if (line[prev] == '"') ++prev; + if (line[curr - 1] == '"') --curr; + out.push_back(line.substr(prev, curr - prev)); prev = pos + 1; pos = line.find_first_of(',', pos + 1); curr = pos; } curr = line.size(); - if(line[prev] == '"') ++prev; - if(line[curr-1] == '"') --curr; - out.push_back(line.substr(prev, curr-prev)); + if (line[prev] == '"') ++prev; + if (line[curr - 1] == '"') --curr; + out.push_back(line.substr(prev, curr - prev)); return out; } } // end namespace internal -size_t AddChecker(const char* bm_name, ResultsCheckFn fn) -{ - auto &rc = internal::GetResultsChecker(); +size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { + auto& rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); } int Results::NumThreads() const { auto pos = name.find("/threads:"); - if(pos == name.npos) return 1; + if (pos == name.npos) return 1; auto end = name.find('/', pos + 9); std::stringstream ss; ss << name.substr(pos + 9, end); @@ -305,17 +303,17 @@ int Results::NumThreads() const { double Results::GetTime(BenchmarkTime which) const { CHECK(which == kCpuTime || which == kRealTime); - const char *which_str = which == kCpuTime ? "cpu_time" : "real_time"; - double val = GetAs< double >(which_str); + const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; + double val = GetAs(which_str); auto unit = Get("time_unit"); CHECK(unit); - if(*unit == "ns") { + if (*unit == "ns") { return val * 1.e-9; - } else if(*unit == "us") { + } else if (*unit == "us") { return val * 1.e-6; - } else if(*unit == "ms") { + } else if (*unit == "ms") { return val * 1.e-3; - } else if(*unit == "s") { + } else if (*unit == "s") { return val; } else { CHECK(1 == 0) << "unknown time unit: " << *unit; @@ -333,7 +331,7 @@ TestCase::TestCase(std::string re, int rule) substituted_regex(internal::PerformSubstitutions(regex_str)), regex(std::make_shared()) { std::string err_str; - regex->Init(substituted_regex,& err_str); + regex->Init(substituted_regex, &err_str); CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex << "\"" << "\n originally \"" << regex_str << "\"" @@ -367,7 +365,7 @@ int SetSubstitutions( void RunOutputTests(int argc, char* argv[]) { using internal::GetTestCaseList; benchmark::Initialize(&argc, argv); - auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true); + auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true); benchmark::ConsoleReporter CR(options); benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; @@ -416,7 +414,7 @@ void RunOutputTests(int argc, char* argv[]) { // now that we know the output is as expected, we can dispatch // the checks to subscribees. - auto &csv = TestCases[2]; + auto& csv = TestCases[2]; // would use == but gcc spits a warning CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc index 8ab2c29939..18de6d68e2 100644 --- a/test/register_benchmark_test.cc +++ b/test/register_benchmark_test.cc @@ -29,6 +29,7 @@ struct TestCase { typedef benchmark::BenchmarkReporter::Run Run; void CheckRun(Run const& run) const { + // clang-format off CHECK(name == run.benchmark_name) << "expected " << name << " got " << run.benchmark_name; if (label) { @@ -37,6 +38,7 @@ struct TestCase { } else { CHECK(run.report_label == ""); } + // clang-format on } }; diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 23eb1baf63..1662fcb8b5 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -9,10 +9,9 @@ // ---------------------- Testing Prologue Output -------------------------- // // ========================================================================= // -ADD_CASES(TC_ConsoleOut, - {{"^[-]+$", MR_Next}, - {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, - {"^[-]+$", MR_Next}}); +ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, + {"^[-]+$", MR_Next}}); static int AddContextCases() { AddCases(TC_ConsoleErr, { @@ -20,14 +19,15 @@ static int AddContextCases() { {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s\\)", MR_Next}, }); - AddCases(TC_JSONOut, {{"^\\{", MR_Default}, - {"\"context\":", MR_Next}, - {"\"date\": \"", MR_Next}, - {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next}, - {"\"num_cpus\": %int,$", MR_Next}, - {"\"mhz_per_cpu\": %float,$", MR_Next}, - {"\"cpu_scaling_enabled\": ", MR_Next}, - {"\"caches\": \\[$", MR_Next}}); + AddCases(TC_JSONOut, + {{"^\\{", MR_Default}, + {"\"context\":", MR_Next}, + {"\"date\": \"", MR_Next}, + {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next}, + {"\"num_cpus\": %int,$", MR_Next}, + {"\"mhz_per_cpu\": %float,$", MR_Next}, + {"\"cpu_scaling_enabled\": ", MR_Next}, + {"\"caches\": \\[$", MR_Next}}); auto const& Caches = benchmark::CPUInfo::Get().caches; if (!Caches.empty()) { AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}}); @@ -348,9 +348,12 @@ void BM_UserStats(benchmark::State& state) { for (auto _ : state) { } } +// clang-format off BENCHMARK(BM_UserStats) - ->Repetitions(3) - ->ComputeStatistics("", UserStatistics); + ->Repetitions(3) + ->ComputeStatistics("", UserStatistics); +// clang-format on + // check that user-provided stats is calculated, and is after the default-ones // empty string as name is intentional, it would sort before anything else ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 8d2c342a9a..39785fb7f6 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -33,8 +33,8 @@ struct TestCase { typedef benchmark::BenchmarkReporter::Run Run; void CheckRun(Run const& run) const { - CHECK(name == run.benchmark_name) << "expected " << name << " got " - << run.benchmark_name; + CHECK(name == run.benchmark_name) + << "expected " << name << " got " << run.benchmark_name; CHECK(error_occurred == run.error_occurred); CHECK(error_message == run.error_message); if (error_occurred) { @@ -70,7 +70,6 @@ void BM_error_before_running(benchmark::State& state) { BENCHMARK(BM_error_before_running); ADD_CASES("BM_error_before_running", {{"", true, "error message"}}); - void BM_error_before_running_batch(benchmark::State& state) { state.SkipWithError("error message"); while (state.KeepRunningBatch(17)) { @@ -124,7 +123,7 @@ void BM_error_during_running_ranged_for(benchmark::State& state) { // Test the unfortunate but documented behavior that the ranged-for loop // doesn't automatically terminate when SkipWithError is set. assert(++It != End); - break; // Required behavior + break; // Required behavior } } } @@ -133,8 +132,6 @@ ADD_CASES("BM_error_during_running_ranged_for", {{"/1/iterations:5", true, "error message"}, {"/2/iterations:5", false, ""}}); - - void BM_error_after_running(benchmark::State& state) { for (auto _ : state) { benchmark::DoNotOptimize(state.iterations()); diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc index e2c5c8648d..abe9a4ddb5 100644 --- a/test/state_assembly_test.cc +++ b/test/state_assembly_test.cc @@ -4,11 +4,13 @@ #pragma clang diagnostic ignored "-Wreturn-type" #endif +// clang-format off extern "C" { extern int ExternInt; benchmark::State& GetState(); void Fn(); } +// clang-format on using benchmark::State; diff --git a/test/statistics_gtest.cc b/test/statistics_gtest.cc index e57460f75c..99e314920c 100644 --- a/test/statistics_gtest.cc +++ b/test/statistics_gtest.cc @@ -7,22 +7,22 @@ namespace { TEST(StatisticsTest, Mean) { - EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42,42,42,42}), 42.0); - EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1,2,3,4}), 2.5); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({42, 42, 42, 42}), 42.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 3, 4}), 2.5); EXPECT_DOUBLE_EQ(benchmark::StatisticsMean({1, 2, 5, 10, 10, 14}), 7.0); } TEST(StatisticsTest, Median) { - EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42,42,42,42}), 42.0); - EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1,2,3,4}), 2.5); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({42, 42, 42, 42}), 42.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 3, 4}), 2.5); EXPECT_DOUBLE_EQ(benchmark::StatisticsMedian({1, 2, 5, 10, 10}), 5.0); } TEST(StatisticsTest, StdDev) { EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0); - EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1,2,3}), 1.0); + EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0); EXPECT_FLOAT_EQ(benchmark::StatisticsStdDev({1.5, 2.4, 3.3, 4.2, 5.1}), - 1.42302495); + 1.42302495); } } // end namespace diff --git a/test/templated_fixture_test.cc b/test/templated_fixture_test.cc index ec5b4c0cc0..fe9865cc77 100644 --- a/test/templated_fixture_test.cc +++ b/test/templated_fixture_test.cc @@ -4,15 +4,15 @@ #include #include -template +template class MyFixture : public ::benchmark::Fixture { -public: + public: MyFixture() : data(0) {} T data; }; -BENCHMARK_TEMPLATE_F(MyFixture, Foo, int)(benchmark::State &st) { +BENCHMARK_TEMPLATE_F(MyFixture, Foo, int)(benchmark::State& st) { for (auto _ : st) { data += 1; } diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 9b8a6132e6..4f126b6d97 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -7,9 +7,11 @@ // @todo: this checks the full output at once; the rule for // CounterSet1 was failing because it was not matching "^[-]+$". // @todo: check that the counters are vertically aligned. -ADD_CASES(TC_ConsoleOut, { -// keeping these lines long improves readability, so: -// clang-format off +ADD_CASES( + TC_ConsoleOut, + { + // keeping these lines long improves readability, so: + // clang-format off {"^[-]+$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, {"^[-]+$", MR_Next}, @@ -44,8 +46,8 @@ ADD_CASES(TC_ConsoleOut, { {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}, -// clang-format on -}); + // clang-format on + }); ADD_CASES(TC_CSVOut, {{"%csv_header," "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); @@ -58,12 +60,12 @@ void BM_Counters_Tabular(benchmark::State& state) { } namespace bm = benchmark; state.counters.insert({ - {"Foo", { 1, bm::Counter::kAvgThreads}}, - {"Bar", { 2, bm::Counter::kAvgThreads}}, - {"Baz", { 4, bm::Counter::kAvgThreads}}, - {"Bat", { 8, bm::Counter::kAvgThreads}}, - {"Frob", {16, bm::Counter::kAvgThreads}}, - {"Lob", {32, bm::Counter::kAvgThreads}}, + {"Foo", {1, bm::Counter::kAvgThreads}}, + {"Bar", {2, bm::Counter::kAvgThreads}}, + {"Baz", {4, bm::Counter::kAvgThreads}}, + {"Bat", {8, bm::Counter::kAvgThreads}}, + {"Frob", {16, bm::Counter::kAvgThreads}}, + {"Lob", {32, bm::Counter::kAvgThreads}}, }); } BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); @@ -102,12 +104,12 @@ void BM_CounterRates_Tabular(benchmark::State& state) { } namespace bm = benchmark; state.counters.insert({ - {"Foo", { 1, bm::Counter::kAvgThreadsRate}}, - {"Bar", { 2, bm::Counter::kAvgThreadsRate}}, - {"Baz", { 4, bm::Counter::kAvgThreadsRate}}, - {"Bat", { 8, bm::Counter::kAvgThreadsRate}}, - {"Frob", {16, bm::Counter::kAvgThreadsRate}}, - {"Lob", {32, bm::Counter::kAvgThreadsRate}}, + {"Foo", {1, bm::Counter::kAvgThreadsRate}}, + {"Bar", {2, bm::Counter::kAvgThreadsRate}}, + {"Baz", {4, bm::Counter::kAvgThreadsRate}}, + {"Bat", {8, bm::Counter::kAvgThreadsRate}}, + {"Frob", {16, bm::Counter::kAvgThreadsRate}}, + {"Lob", {32, bm::Counter::kAvgThreadsRate}}, }); } BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); @@ -129,12 +131,12 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," // to CHECK_BENCHMARK_RESULTS() void CheckTabularRate(Results const& e) { double t = e.DurationCPUTime(); - CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001); } CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int", &CheckTabularRate); @@ -149,9 +151,9 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { } namespace bm = benchmark; state.counters.insert({ - {"Foo", {10, bm::Counter::kAvgThreads}}, - {"Bar", {20, bm::Counter::kAvgThreads}}, - {"Baz", {40, bm::Counter::kAvgThreads}}, + {"Foo", {10, bm::Counter::kAvgThreads}}, + {"Bar", {20, bm::Counter::kAvgThreads}}, + {"Baz", {40, bm::Counter::kAvgThreads}}, }); } BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); @@ -181,9 +183,9 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { } namespace bm = benchmark; state.counters.insert({ - {"Foo", {15, bm::Counter::kAvgThreads}}, - {"Bar", {25, bm::Counter::kAvgThreads}}, - {"Baz", {45, bm::Counter::kAvgThreads}}, + {"Foo", {15, bm::Counter::kAvgThreads}}, + {"Bar", {25, bm::Counter::kAvgThreads}}, + {"Baz", {45, bm::Counter::kAvgThreads}}, }); } BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); @@ -217,9 +219,9 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { } namespace bm = benchmark; state.counters.insert({ - {"Foo", {10, bm::Counter::kAvgThreads}}, - {"Bat", {30, bm::Counter::kAvgThreads}}, - {"Baz", {40, bm::Counter::kAvgThreads}}, + {"Foo", {10, bm::Counter::kAvgThreads}}, + {"Bat", {30, bm::Counter::kAvgThreads}}, + {"Baz", {40, bm::Counter::kAvgThreads}}, }); } BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 06aafb1fa1..acc8426485 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -8,12 +8,16 @@ // ---------------------- Testing Prologue Output -------------------------- // // ========================================================================= // +// clang-format off + ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, {"^[-]+$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); +// clang-format on + // ========================================================================= // // ------------------------- Simple Counters Output ------------------------ // // ========================================================================= // @@ -25,7 +29,8 @@ void BM_Counters_Simple(benchmark::State& state) { state.counters["bar"] = 2 * (double)state.iterations(); } BENCHMARK(BM_Counters_Simple); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -38,10 +43,10 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckSimple(Results const& e) { - double its = e.GetAs< double >("iterations"); + double its = e.GetAs("iterations"); CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); // check that the value of bar is within 0.1% of the expected value - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2.*its, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); @@ -49,7 +54,9 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); // --------------------- Counters+Items+Bytes/s Output --------------------- // // ========================================================================= // -namespace { int num_calls1 = 0; } +namespace { +int num_calls1 = 0; +} void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { } @@ -77,12 +84,12 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckBytesAndItemsPSec(Results const& e) { - double t = e.DurationCPUTime(); // this (and not real time) is the time used + double t = e.DurationCPUTime(); // this (and not real time) is the time used CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); // check that the values are within 0.1% of the expected values - CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364./t, 0.001); - CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150./t, 0.001); + CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364. / t, 0.001); + CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150. / t, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", &CheckBytesAndItemsPSec); @@ -99,7 +106,9 @@ void BM_Counters_Rate(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; } BENCHMARK(BM_Counters_Rate); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES( + TC_ConsoleOut, + {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -112,10 +121,10 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckRate(Results const& e) { - double t = e.DurationCPUTime(); // this (and not real time) is the time used + double t = e.DurationCPUTime(); // this (and not real time) is the time used // check that the values are within 0.1% of the expected values - CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./t, 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / t, 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); @@ -130,7 +139,8 @@ void BM_Counters_Threads(benchmark::State& state) { state.counters["bar"] = 2; } BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -139,7 +149,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"bar\": %float,$", MR_Next}, {"\"foo\": %float$", MR_Next}, {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckThreads(Results const& e) { @@ -160,7 +172,8 @@ void BM_Counters_AvgThreads(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; } BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " + "%console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -169,7 +182,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"bar\": %float,$", MR_Next}, {"\"foo\": %float$", MR_Next}, {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckAvgThreads(Results const& e) { @@ -191,21 +206,24 @@ void BM_Counters_AvgThreadsRate(benchmark::State& state) { state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; } BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); -ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/threads:%int\",%csv_report,%float,%float$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/" + "threads:%int\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckAvgThreadsRate(Results const& e) { - CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./e.DurationCPUTime(), 0.001); - CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./e.DurationCPUTime(), 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / e.DurationCPUTime(), 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / e.DurationCPUTime(), 0.001); } CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", &CheckAvgThreadsRate); From 7fb3c564e51ce3aa6100484a0b25c603ea5fd123 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Tue, 5 Jun 2018 03:36:26 -0700 Subject: [PATCH 052/330] Fix compilation on Android with GNU STL (#596) * Fix compilation on Android with GNU STL GNU STL in Android NDK lacks string conversion functions from C++11, including std::stoul, std::stoi, and std::stod. This patch reimplements these functions in benchmark:: namespace using C-style equivalents from C++03. * Avoid use of log2 which doesn't exist in Android GNU STL GNU STL in Android NDK lacks log2 function from C99/C++11. This patch replaces their use in the code with double log(double) function. --- src/complexity.cc | 7 +- src/internal_macros.h | 7 ++ src/string_util.cc | 89 +++++++++++++++++++++++ src/string_util.h | 17 +++++ src/sysinfo.cc | 8 +-- test/CMakeLists.txt | 1 + test/complexity_test.cc | 3 +- test/string_util_gtest.cc | 146 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 271 insertions(+), 7 deletions(-) create mode 100644 test/string_util_gtest.cc diff --git a/src/complexity.cc b/src/complexity.cc index 97bf6e09b3..02fe3feb3c 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -26,6 +26,7 @@ namespace benchmark { // Internal function to calculate the different scalability forms BigOFunc* FittingCurve(BigO complexity) { + static const double kLog2E = 1.44269504088896340736; switch (complexity) { case oN: return [](int64_t n) -> double { return static_cast(n); }; @@ -34,9 +35,11 @@ BigOFunc* FittingCurve(BigO complexity) { case oNCubed: return [](int64_t n) -> double { return std::pow(n, 3); }; case oLogN: - return [](int64_t n) { return log2(n); }; + /* Note: can't use log2 because Android's GNU STL lacks it */ + return [](int64_t n) { return kLog2E * log(n); }; case oNLogN: - return [](int64_t n) { return n * log2(n); }; + /* Note: can't use log2 because Android's GNU STL lacks it */ + return [](int64_t n) { return kLog2E * n * log(n); }; case o1: default: return [](int64_t) { return 1.0; }; diff --git a/src/internal_macros.h b/src/internal_macros.h index 500f0ababc..b7e9203ff6 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -3,6 +3,9 @@ #include "benchmark/benchmark.h" +/* Needed to detect STL */ +#include + // clang-format off #ifndef __has_feature @@ -69,6 +72,10 @@ #define BENCHMARK_OS_SOLARIS 1 #endif +#if defined(__ANDROID__) && defined(__GLIBCXX__) +#define BENCHMARK_STL_ANDROID_GNUSTL 1 +#endif + #if !__has_feature(cxx_exceptions) && !defined(__cpp_exceptions) \ && !defined(__EXCEPTIONS) #define BENCHMARK_HAS_NO_EXCEPTIONS diff --git a/src/string_util.cc b/src/string_util.cc index ebc3acebd2..05ac5b4ea3 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -169,4 +169,93 @@ void ReplaceAll(std::string* str, const std::string& from, } } +#ifdef BENCHMARK_STL_ANDROID_GNUSTL +/* + * GNU STL in Android NDK lacks support for some C++11 functions, including + * stoul, stoi, stod. We reimplement them here using C functions strtoul, + * strtol, strtod. Note that reimplemented functions are in benchmark:: + * namespace, not std:: namespace. + */ +unsigned long stoul(const std::string& str, size_t* pos, int base) { + /* Record previous errno */ + const int oldErrno = errno; + errno = 0; + + const char* strStart = str.c_str(); + char* strEnd = const_cast(strStart); + const unsigned long result = strtoul(strStart, &strEnd, base); + + const int strtoulErrno = errno; + /* Restore previous errno */ + errno = oldErrno; + + /* Check for errors and return */ + if (strtoulErrno == ERANGE) { + throw std::out_of_range( + "stoul failed: " + str + " is outside of range of unsigned long"); + } else if (strEnd == strStart || strtoulErrno != 0) { + throw std::invalid_argument( + "stoul failed: " + str + " is not an integer"); + } + if (pos != nullptr) { + *pos = static_cast(strEnd - strStart); + } + return result; +} + +int stoi(const std::string& str, size_t* pos, int base) { + /* Record previous errno */ + const int oldErrno = errno; + errno = 0; + + const char* strStart = str.c_str(); + char* strEnd = const_cast(strStart); + const long result = strtol(strStart, &strEnd, base); + + const int strtolErrno = errno; + /* Restore previous errno */ + errno = oldErrno; + + /* Check for errors and return */ + if (strtolErrno == ERANGE || long(int(result)) != result) { + throw std::out_of_range( + "stoul failed: " + str + " is outside of range of int"); + } else if (strEnd == strStart || strtolErrno != 0) { + throw std::invalid_argument( + "stoul failed: " + str + " is not an integer"); + } + if (pos != nullptr) { + *pos = static_cast(strEnd - strStart); + } + return int(result); +} + +double stod(const std::string& str, size_t* pos) { + /* Record previous errno */ + const int oldErrno = errno; + errno = 0; + + const char* strStart = str.c_str(); + char* strEnd = const_cast(strStart); + const double result = strtod(strStart, &strEnd); + + /* Restore previous errno */ + const int strtodErrno = errno; + errno = oldErrno; + + /* Check for errors and return */ + if (strtodErrno == ERANGE) { + throw std::out_of_range( + "stoul failed: " + str + " is outside of range of int"); + } else if (strEnd == strStart || strtodErrno != 0) { + throw std::invalid_argument( + "stoul failed: " + str + " is not an integer"); + } + if (pos != nullptr) { + *pos = static_cast(strEnd - strStart); + } + return result; +} +#endif + } // end namespace benchmark diff --git a/src/string_util.h b/src/string_util.h index ca49c4fde1..4a5501273c 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -34,6 +34,23 @@ inline std::string StrCat(Args&&... args) { void ReplaceAll(std::string* str, const std::string& from, const std::string& to); +#ifdef BENCHMARK_STL_ANDROID_GNUSTL +/* + * GNU STL in Android NDK lacks support for some C++11 functions, including + * stoul, stoi, stod. We reimplement them here using C functions strtoul, + * strtol, strtod. Note that reimplemented functions are in benchmark:: + * namespace, not std:: namespace. + */ +unsigned long stoul(const std::string& str, size_t* pos = nullptr, + int base = 10); +int stoi(const std::string& str, size_t* pos = nullptr, int base = 10); +double stod(const std::string& str, size_t* pos = nullptr); +#else +using std::stoul; +using std::stoi; +using std::stod; +#endif + } // end namespace benchmark #endif // BENCHMARK_STRING_UTIL_H_ diff --git a/src/sysinfo.cc b/src/sysinfo.cc index d19d0ef4c1..73064b97ba 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -225,7 +225,7 @@ int CountSetBitsInCPUMap(std::string Val) { auto CountBits = [](std::string Part) { using CPUMask = std::bitset; Part = "0x" + Part; - CPUMask Mask(std::stoul(Part, nullptr, 16)); + CPUMask Mask(benchmark::stoul(Part, nullptr, 16)); return static_cast(Mask.count()); }; size_t Pos; @@ -408,7 +408,7 @@ int GetNumCPUs() { if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { NumCPUs++; if (!value.empty()) { - int CurID = std::stoi(value); + int CurID = benchmark::stoi(value); MaxID = std::max(CurID, MaxID); } } @@ -481,12 +481,12 @@ double GetCPUCyclesPerSecond() { // which would cause infinite looping in WallTime_Init. if (startsWithKey(ln, "cpu MHz")) { if (!value.empty()) { - double cycles_per_second = std::stod(value) * 1000000.0; + double cycles_per_second = benchmark::stod(value) * 1000000.0; if (cycles_per_second > 0) return cycles_per_second; } } else if (startsWithKey(ln, "bogomips")) { if (!value.empty()) { - bogo_clock = std::stod(value) * 1000000.0; + bogo_clock = benchmark::stod(value) * 1000000.0; if (bogo_clock < 0.0) bogo_clock = error_value; } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 05ae804bfe..f49ca5148f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -180,6 +180,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(benchmark_gtest) add_gtest(statistics_gtest) + add_gtest(string_util_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/test/complexity_test.cc b/test/complexity_test.cc index ab832861ec..c732e6ed49 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -134,6 +134,7 @@ static void BM_Complexity_O_N_log_N(benchmark::State& state) { } state.SetComplexityN(state.range(0)); } +static const double kLog2E = 1.44269504088896340736; BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -141,7 +142,7 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) { return n * log2(n); }); + ->Complexity([](int64_t n) { return kLog2E * n * log(n); }); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc new file mode 100644 index 0000000000..4c81734cf8 --- /dev/null +++ b/test/string_util_gtest.cc @@ -0,0 +1,146 @@ +//===---------------------------------------------------------------------===// +// statistics_test - Unit tests for src/statistics.cc +//===---------------------------------------------------------------------===// + +#include "../src/string_util.h" +#include "gtest/gtest.h" + +namespace { +TEST(StringUtilTest, stoul) { + { + size_t pos = 0; + EXPECT_EQ(0, benchmark::stoul("0", &pos)); + EXPECT_EQ(1, pos); + } + { + size_t pos = 0; + EXPECT_EQ(7, benchmark::stoul("7", &pos)); + EXPECT_EQ(1, pos); + } + { + size_t pos = 0; + EXPECT_EQ(135, benchmark::stoul("135", &pos)); + EXPECT_EQ(3, pos); + } +#if ULONG_MAX == 0xFFFFFFFFul + { + size_t pos = 0; + EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos)); + EXPECT_EQ(10, pos); + } +#elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul + { + size_t pos = 0; + EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, benchmark::stoul("18446744073709551615", &pos)); + EXPECT_EQ(20, pos); + } +#endif + { + size_t pos = 0; + EXPECT_EQ(10, benchmark::stoul("1010", &pos, 2)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(520, benchmark::stoul("1010", &pos, 8)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1010, benchmark::stoul("1010", &pos, 10)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(4112, benchmark::stoul("1010", &pos, 16)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(0xBEEF, benchmark::stoul("BEEF", &pos, 16)); + EXPECT_EQ(4, pos); + } + { + ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); + } +} + +TEST(StringUtilTest, stoi) { + { + size_t pos = 0; + EXPECT_EQ(0, benchmark::stoi("0", &pos)); + EXPECT_EQ(1, pos); + } + { + size_t pos = 0; + EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); + EXPECT_EQ(3, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); + EXPECT_EQ(4, pos); + } + { + ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); + } +} + +TEST(StringUtilTest, stod) { + { + size_t pos = 0; + EXPECT_EQ(0.0, benchmark::stod("0", &pos)); + EXPECT_EQ(1, pos); + } + { + size_t pos = 0; + EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); + EXPECT_EQ(3, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); + EXPECT_EQ(4, pos); + } + { + size_t pos = 0; + EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); + EXPECT_EQ(3, pos); + } + { + size_t pos = 0; + /* Note: exactly representable as double */ + EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); + EXPECT_EQ(8, pos); + } + { + ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); + } +} + +} // end namespace From 1301f53e3173e5e8c1583b9b57a34bbbd5970366 Mon Sep 17 00:00:00 2001 From: Sergiu Deitsch Date: Tue, 5 Jun 2018 16:01:44 +0200 Subject: [PATCH 053/330] cmake: use numeric version in package config (#611) --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 701804ba0e..c75c0956f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,7 +69,7 @@ set(namespace "${PROJECT_NAME}::") include(CMakePackageConfigHelpers) write_basic_package_version_file( - "${version_config}" VERSION ${GIT_VERSION} COMPATIBILITY SameMajorVersion + "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion ) configure_file("${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in" "${project_config}" @ONLY) From 505be96ab23056580a3a2315abba048f4428b04e Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Wed, 6 Jun 2018 04:32:42 -0700 Subject: [PATCH 054/330] Avoid using CMake 3.6 feature list(FILTER ...) (#612) list(FILTER ...) is a CMake 3.6 feature, but benchmark targets CMake 2.8.12 --- src/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c75c0956f4..977474f43f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,7 +11,10 @@ file(GLOB *.cc ${PROJECT_SOURCE_DIR}/include/benchmark/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.h) -list(FILTER SOURCE_FILES EXCLUDE REGEX "benchmark_main\\.cc") +file(GLOB BENCHMARK_MAIN "benchmark_main.cc") +foreach(item ${BENCHMARK_MAIN}) + list(REMOVE_ITEM SOURCE_FILES "${item}") +endforeach() add_library(benchmark ${SOURCE_FILES}) set_target_properties(benchmark PROPERTIES From 151ead6242b2075b5a3d55905440a3aab245a800 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 7 Jun 2018 12:54:14 +0100 Subject: [PATCH 055/330] Disable deprecation warnings when -Werror is enabled. (#609) Fixes #608 --- CMakeLists.txt | 4 ++++ include/benchmark/benchmark.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f647c5999c..92745fdb7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,6 +143,10 @@ else() # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) + # Disable deprecation warnings for release builds (when -Werror is enabled). + add_cxx_compiler_flag(-Wno-deprecated RELEASE) + add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) + add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 58222508b3..be9498cb24 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -226,7 +226,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_INTERNAL_TOSTRING2(x) #x #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) #else From 7d03f2df490c89b2a2055e9be4e2c36db5aedd80 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 18 Jun 2018 14:58:16 +0300 Subject: [PATCH 056/330] [Tooling] Enable U Test by default, add tooltip about repetition count. (#617) As previously discussed, let's flip the switch ^^. This exposes the problem that it will now be run for everyone, even if one did not read the help about the recommended repetition count. This is not good. So i think we can do the smart thing: ``` $ ./compare.py benchmarks gbench/Inputs/test3_run{0,1}.json Comparing gbench/Inputs/test3_run0.json to gbench/Inputs/test3_run1.json Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------- BM_One -0.1000 +0.1000 10 9 100 110 BM_Two +0.1111 -0.0111 9 10 90 89 BM_Two +0.2500 +0.1125 8 10 80 89 BM_Two_pvalue 0.2207 0.6831 U Test, Repetitions: 2. WARNING: Results unreliable! 9+ repetitions recommended. BM_Two_stat +0.0000 +0.0000 8 8 80 80 ``` (old screenshot) ![image](https://user-images.githubusercontent.com/88600/41502182-ea25d872-71bc-11e8-9842-8aa049509b14.png) Or, in the good case (noise omitted): ``` s$ ./compare.py benchmarks /tmp/run{0,1}.json Comparing /tmp/run0.json to /tmp/run1.json Benchmark Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- <99 more rows like this> ./_T012014.RW2/threads:8/real_time +0.0160 +0.0596 46 47 10 10 ./_T012014.RW2/threads:8/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 100 ./_T012014.RW2/threads:8/real_time_mean +0.0094 +0.0609 46 47 10 10 ./_T012014.RW2/threads:8/real_time_median +0.0104 +0.0613 46 46 10 10 ./_T012014.RW2/threads:8/real_time_stddev -0.1160 -0.1807 1 1 0 0 ``` (old screenshot) ![image](https://user-images.githubusercontent.com/88600/41502185-fb8193f4-71bc-11e8-85fa-cbba83e39db4.png) --- tools/compare.py | 45 +++++++++++----------- tools/gbench/Inputs/test3_run0.json | 2 +- tools/gbench/Inputs/test3_run1.json | 2 +- tools/gbench/report.py | 58 +++++++++++++++++++++-------- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/tools/compare.py b/tools/compare.py index f293306eac..d27e24b349 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -38,10 +38,11 @@ def create_parser(): utest = parser.add_argument_group() utest.add_argument( - '-u', - '--utest', - action="store_true", - help="Do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than 9) number of repetitions to be meaningful!") + '--no-utest', + dest='utest', + default=True, + action="store_false", + help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS)) alpha_default = 0.05 utest.add_argument( "--alpha", @@ -245,36 +246,36 @@ def setUp(self): def test_benchmarks_basic(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1]) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) - def test_benchmarks_basic_with_utest(self): + def test_benchmarks_basic_without_utest(self): parsed = self.parser.parse_args( - ['-u', 'benchmarks', self.testInput0, self.testInput1]) - self.assertTrue(parsed.utest) + ['--no-utest', 'benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.05) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) - def test_benchmarks_basic_with_utest(self): + def test_benchmarks_basic_with_utest_alpha(self): parsed = self.parser.parse_args( - ['--utest', 'benchmarks', self.testInput0, self.testInput1]) + ['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) self.assertTrue(parsed.utest) - self.assertEqual(parsed.utest_alpha, 0.05) + self.assertEqual(parsed.utest_alpha, 0.314) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) - def test_benchmarks_basic_with_utest_alpha(self): + def test_benchmarks_basic_without_utest_with_utest_alpha(self): parsed = self.parser.parse_args( - ['--utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) - self.assertTrue(parsed.utest) + ['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.314) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -284,7 +285,7 @@ def test_benchmarks_basic_with_utest_alpha(self): def test_benchmarks_with_remainder(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, 'd']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) @@ -293,7 +294,7 @@ def test_benchmarks_with_remainder(self): def test_benchmarks_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, '--', 'e']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.test_contender[0].name, self.testInput1) @@ -302,7 +303,7 @@ def test_benchmarks_with_remainder_after_doubleminus(self): def test_filters_basic(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -312,7 +313,7 @@ def test_filters_basic(self): def test_filters_with_remainder(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', 'e']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -322,7 +323,7 @@ def test_filters_with_remainder(self): def test_filters_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', '--', 'f']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -332,7 +333,7 @@ def test_filters_with_remainder_after_doubleminus(self): def test_benchmarksfiltered_basic(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -343,7 +344,7 @@ def test_benchmarksfiltered_basic(self): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') @@ -354,7 +355,7 @@ def test_benchmarksfiltered_with_remainder(self): def test_benchmarksfiltered_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g']) - self.assertFalse(parsed.utest) + self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) self.assertEqual(parsed.filter_baseline[0], 'c') diff --git a/tools/gbench/Inputs/test3_run0.json b/tools/gbench/Inputs/test3_run0.json index c777bb0ca9..ca793f3367 100644 --- a/tools/gbench/Inputs/test3_run0.json +++ b/tools/gbench/Inputs/test3_run0.json @@ -29,7 +29,7 @@ "time_unit": "ns" }, { - "name": "BM_Two_stat", + "name": "short", "iterations": 1000, "real_time": 8, "cpu_time": 80, diff --git a/tools/gbench/Inputs/test3_run1.json b/tools/gbench/Inputs/test3_run1.json index 035033391a..e5cf50c744 100644 --- a/tools/gbench/Inputs/test3_run1.json +++ b/tools/gbench/Inputs/test3_run1.json @@ -29,7 +29,7 @@ "time_unit": "ns" }, { - "name": "BM_Two_stat", + "name": "short", "iterations": 1000, "real_time": 8, "cpu_time": 80, diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 4cdd3b74c1..4d03a54767 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -34,6 +34,9 @@ def __format__(self, format): BC_BOLD = BenchmarkColor('BOLD', '\033[1m') BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m') +UTEST_MIN_REPETITIONS = 2 +UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. + def color_format(use_color, fmt_str, *args, **kwargs): """ @@ -109,11 +112,11 @@ def find_test(name): return b return None - utest_col_name = "U-test (p-value)" + utest_col_name = "_pvalue" first_col_width = max( first_col_width, - len('Benchmark'), - len(utest_col_name)) + len('Benchmark')) + first_col_width += len(utest_col_name) first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format( 'Benchmark', 12 + first_col_width) output_strs = [first_line, '-' * len(first_line)] @@ -126,16 +129,15 @@ def find_test(name): if 'real_time' in bn and 'cpu_time' in bn) for bn in gen: fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" - special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}" + special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}" if last_name is None: last_name = bn['name'] if last_name != bn['name']: - MIN_REPETITIONS = 2 - if ((len(timings_time[0]) >= MIN_REPETITIONS) and - (len(timings_time[1]) >= MIN_REPETITIONS) and - (len(timings_cpu[0]) >= MIN_REPETITIONS) and - (len(timings_cpu[1]) >= MIN_REPETITIONS)): + if ((len(timings_time[0]) >= UTEST_MIN_REPETITIONS) and + (len(timings_time[1]) >= UTEST_MIN_REPETITIONS) and + (len(timings_cpu[0]) >= UTEST_MIN_REPETITIONS) and + (len(timings_cpu[1]) >= UTEST_MIN_REPETITIONS)): if utest: def get_utest_color(pval): if pval >= utest_alpha: @@ -146,15 +148,24 @@ def get_utest_color(pval): timings_time[0], timings_time[1], alternative='two-sided').pvalue cpu_pvalue = mannwhitneyu( timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue + dsc = "U Test, Repetitions: {}".format(len(timings_cpu[0])) + dsc_color = BC_OKGREEN + if len(timings_cpu[0]) < UTEST_OPTIMAL_REPETITIONS: + dsc_color = BC_WARNING + dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( + UTEST_OPTIMAL_REPETITIONS) output_strs += [color_format(use_color, special_str, BC_HEADER, - utest_col_name, + "{}{}".format(last_name, + utest_col_name), first_col_width, get_utest_color(time_pvalue), time_pvalue, get_utest_color(cpu_pvalue), cpu_pvalue, + dsc_color, + dsc, endc=BC_ENDC)] last_name = bn['name'] timings_time = [[], []] @@ -229,9 +240,12 @@ def test_basic(self): ['BM_1PercentSlower', '+0.0100', '+0.0100', '100', '101', '100', '101'], ['BM_10PercentFaster', '-0.1000', '-0.1000', '100', '90', '100', '90'], ['BM_10PercentSlower', '+0.1000', '+0.1000', '100', '110', '100', '110'], - ['BM_100xSlower', '+99.0000', '+99.0000', '100', '10000', '100', '10000'], - ['BM_100xFaster', '-0.9900', '-0.9900', '10000', '100', '10000', '100'], - ['BM_10PercentCPUToTime', '+0.1000', '-0.1000', '100', '110', '100', '90'], + ['BM_100xSlower', '+99.0000', '+99.0000', + '100', '10000', '100', '10000'], + ['BM_100xFaster', '-0.9900', '-0.9900', + '10000', '100', '10000', '100'], + ['BM_10PercentCPUToTime', '+0.1000', + '-0.1000', '100', '110', '100', '90'], ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ] @@ -239,6 +253,7 @@ def test_basic(self): output_lines_with_header = generate_difference_report( json1, json2, use_color=False) output_lines = output_lines_with_header[2:] + print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) for i in range(0, len(output_lines)): @@ -302,13 +317,26 @@ def test_utest(self): ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'], - ['U-test', '(p-value)', '0.2207', '0.6831'], - ['BM_Two_stat', '+0.0000', '+0.0000', '8', '8', '80', '80'], + ['BM_Two_pvalue', + '0.2207', + '0.6831', + 'U', + 'Test,', + 'Repetitions:', + '2.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], + ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'], ] json1, json2 = self.load_results() output_lines_with_header = generate_difference_report( json1, json2, True, 0.05, use_color=False) output_lines = output_lines_with_header[2:] + print("\n") print("\n".join(output_lines_with_header)) self.assertEqual(len(output_lines), len(expect_lines)) for i in range(0, len(output_lines)): From d8584bda67320937334b038e736f5bffcd438875 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 27 Jun 2018 12:11:30 +0100 Subject: [PATCH 057/330] Use EXPECT_DOUBLE_EQ when comparing doubles in tests. (#624) * Use EXPECT_DOUBLE_EQ when comparing doubles in tests. Fixes #623 * disable 'float-equal' warning --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92745fdb7f..c7fe7a8ed4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,7 +138,6 @@ else() add_cxx_compiler_flag(-pedantic) add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-Wshorten-64-to-32) - add_cxx_compiler_flag(-Wfloat-equal) add_cxx_compiler_flag(-fstrict-aliasing) # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. From b123abdcf4a004ffadca7834d614bdeb36c843c0 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 27 Jun 2018 17:45:30 +0300 Subject: [PATCH 058/330] Add Iteration-related Counter::Flags. Fixes #618 (#621) Inspired by these [two](https://github.com/darktable-org/rawspeed/commit/a1ebe07bea5738f8607b48a7596c172be249590e) [bugs](https://github.com/darktable-org/rawspeed/commit/0891555be56b24f9f4af716604cedfa0da1efc6b) in my code due to the lack of those i have found fixed in my code: * `kIsIterationInvariant` - `* state.iterations()` The value is constant for every iteration, and needs to be **multiplied** by the iteration count. * `kAvgIterations` - `/ state.iterations()` The is global over all the iterations, and needs to be **divided** by the iteration count. They play nice with `kIsRate`: * `kIsIterationInvariantRate` * `kAvgIterationsRate`. I'm not sure how meaningful they are when combined with `kAvgThreads`. I guess the `kIsThreadInvariant` can be added, too, for symmetry with `kAvgThreads`. --- .gitignore | 1 + include/benchmark/benchmark.h | 26 +++++- src/benchmark.cc | 2 +- src/counter.cc | 13 ++- src/counter.h | 2 +- test/output_test.h | 6 +- test/output_test_helper.cc | 4 + test/user_counters_test.cc | 147 +++++++++++++++++++++++++++++++++- 8 files changed, 190 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 050e46987f..8c30e28f53 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ bazel-* # out-of-source build top-level folders. build/ _build/ +build*/ # in-source dependencies /googletest/ diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index be9498cb24..193fffc4be 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -343,12 +343,24 @@ class Counter { kDefaults = 0, // Mark the counter as a rate. It will be presented divided // by the duration of the benchmark. - kIsRate = 1, + kIsRate = 1U << 0U, // Mark the counter as a thread-average quantity. It will be // presented divided by the number of threads. - kAvgThreads = 2, + kAvgThreads = 1U << 1U, // Mark the counter as a thread-average rate. See above. - kAvgThreadsRate = kIsRate | kAvgThreads + kAvgThreadsRate = kIsRate | kAvgThreads, + // Mark the counter as a constant value, valid/same for *every* iteration. + // When reporting, it will be *multiplied* by the iteration count. + kIsIterationInvariant = 1U << 2U, + // Mark the counter as a constant rate. + // When reporting, it will be *multiplied* by the iteration count + // and then divided by the duration of the benchmark. + kIsIterationInvariantRate = kIsRate | kIsIterationInvariant, + // Mark the counter as a iteration-average quantity. + // It will be presented divided by the number of iterations. + kAvgIterations = 1U << 3U, + // Mark the counter as a iteration-average rate. See above. + kAvgIterationsRate = kIsRate | kAvgIterations }; double value; @@ -361,6 +373,14 @@ class Counter { BENCHMARK_ALWAYS_INLINE operator double&() { return value; } }; +// A helper for user code to create unforeseen combinations of Flags, without +// having to do this cast manually each time, or providing this operator. +Counter::Flags inline operator|(const Counter::Flags& LHS, + const Counter::Flags& RHS) { + return static_cast(static_cast(LHS) | + static_cast(RHS)); +} + // This is the container for the user-defined counters. typedef std::map UserCounters; diff --git a/src/benchmark.cc b/src/benchmark.cc index 9422cdb164..da5532251b 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -150,7 +150,7 @@ BenchmarkReporter::Run CreateRunReport( report.complexity_lambda = b.complexity_lambda; report.statistics = b.statistics; report.counters = results.counters; - internal::Finish(&report.counters, seconds, b.threads); + internal::Finish(&report.counters, results.iterations, seconds, b.threads); } return report; } diff --git a/src/counter.cc b/src/counter.cc index 50d89a6a7a..cb604e060b 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -17,7 +17,8 @@ namespace benchmark { namespace internal { -double Finish(Counter const& c, double cpu_time, double num_threads) { +double Finish(Counter const& c, int64_t iterations, double cpu_time, + double num_threads) { double v = c.value; if (c.flags & Counter::kIsRate) { v /= cpu_time; @@ -25,12 +26,18 @@ double Finish(Counter const& c, double cpu_time, double num_threads) { if (c.flags & Counter::kAvgThreads) { v /= num_threads; } + if (c.flags & Counter::kIsIterationInvariant) { + v *= iterations; + } + if (c.flags & Counter::kAvgIterations) { + v /= iterations; + } return v; } -void Finish(UserCounters* l, double cpu_time, double num_threads) { +void Finish(UserCounters* l, int64_t iterations, double cpu_time, double num_threads) { for (auto& c : *l) { - c.second.value = Finish(c.second, cpu_time, num_threads); + c.second.value = Finish(c.second, iterations, cpu_time, num_threads); } } diff --git a/src/counter.h b/src/counter.h index 299cc0266c..d884e50aa1 100644 --- a/src/counter.h +++ b/src/counter.h @@ -18,7 +18,7 @@ namespace benchmark { // these counter-related functions are hidden to reduce API surface. namespace internal { -void Finish(UserCounters* l, double time, double num_threads); +void Finish(UserCounters* l, int64_t iterations, double time, double num_threads); void Increment(UserCounters* l, UserCounters const& r); bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace internal diff --git a/test/output_test.h b/test/output_test.h index b157fe66a2..31a919991f 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -92,6 +92,8 @@ struct Results { int NumThreads() const; + double NumIterations() const; + typedef enum { kCpuTime, kRealTime } BenchmarkTime; // get cpu_time or real_time in seconds @@ -101,11 +103,11 @@ struct Results { // it is better to use fuzzy float checks for this, as the float // ASCII formatting is lossy. double DurationRealTime() const { - return GetAs("iterations") * GetTime(kRealTime); + return NumIterations() * GetTime(kRealTime); } // get the cpu_time duration of the benchmark in seconds double DurationCPUTime() const { - return GetAs("iterations") * GetTime(kCpuTime); + return NumIterations() * GetTime(kCpuTime); } // get the string for a result by name, or nullptr if the name diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 92c2448d9e..394c4f5d1a 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -301,6 +301,10 @@ int Results::NumThreads() const { return num; } +double Results::NumIterations() const { + return GetAs("iterations"); +} + double Results::GetTime(BenchmarkTime which) const { CHECK(which == kCpuTime || which == kRealTime); const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index acc8426485..7f7ccb9f77 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -43,7 +43,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckSimple(Results const& e) { - double its = e.GetAs("iterations"); + double its = e.NumIterations(); CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); // check that the value of bar is within 0.1% of the expected value CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); @@ -228,6 +228,151 @@ void CheckAvgThreadsRate(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", &CheckAvgThreadsRate); +// ========================================================================= // +// ------------------- IterationInvariant Counters Output ------------------ // +// ========================================================================= // + +void BM_Counters_IterationInvariant(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kIsIterationInvariant}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kIsIterationInvariant}; +} +BENCHMARK(BM_Counters_IterationInvariant); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckIterationInvariant(Results const& e) { + double its = e.NumIterations(); + // check that the values are within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. * its, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", + &CheckIterationInvariant); + +// ========================================================================= // +// ----------------- IterationInvariantRate Counters Output ---------------- // +// ========================================================================= // + +void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = + bm::Counter{1, bm::Counter::kIsIterationInvariantRate}; + state.counters["bar"] = + bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kIsIterationInvariant}; +} +BENCHMARK(BM_Counters_kIsIterationInvariantRate); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kIsIterationInvariantRate\",%csv_report," + "%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckIsIterationInvariantRate(Results const& e) { + double its = e.NumIterations(); + double t = e.DurationCPUTime(); // this (and not real time) is the time used + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, its * 1. / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, its * 2. / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", + &CheckIsIterationInvariantRate); + +// ========================================================================= // +// ------------------- AvgIterations Counters Output ------------------ // +// ========================================================================= // + +void BM_Counters_AvgIterations(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterations}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgIterations}; +} +BENCHMARK(BM_Counters_AvgIterations); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " + "bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgIterations(Results const& e) { + double its = e.NumIterations(); + // check that the values are within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); + +// ========================================================================= // +// ----------------- AvgIterationsRate Counters Output ---------------- // +// ========================================================================= // + +void BM_Counters_kAvgIterationsRate(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; + state.counters["bar"] = + bm::Counter{2, bm::Counter::kIsRate | bm::Counter::kAvgIterations}; +} +BENCHMARK(BM_Counters_kAvgIterationsRate); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " + "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report," + "%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgIterationsRate(Results const& e) { + double its = e.NumIterations(); + double t = e.DurationCPUTime(); // this (and not real time) is the time used + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1. / its / t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2. / its / t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_kAvgIterationsRate", + &CheckAvgIterationsRate); + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // From 847c0069021ade355b7678a305f3ac4e4d6f7e79 Mon Sep 17 00:00:00 2001 From: Yoshinari Takaoka Date: Thu, 28 Jun 2018 18:25:54 +0900 Subject: [PATCH 059/330] fixed Google Test (Primer) Documentation link (#628) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 749039a0ac..80e69f6e10 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ the right to change and break the API at any time. ## Prerequisite knowledge Before attempting to understand this framework one should ideally have some familiarity with the structure and format of the Google Test framework, upon which it is based. Documentation for Google Test, including a "Getting Started" (primer) guide, is available here: -https://github.com/google/googletest/blob/master/googletest/docs/Documentation.md +https://github.com/google/googletest/blob/master/googletest/docs/primer.md ## Example usage From 5946795e82f778fff891c37f56c0d7a76a118bf9 Mon Sep 17 00:00:00 2001 From: Federico Ficarelli Date: Tue, 3 Jul 2018 11:13:22 +0200 Subject: [PATCH 060/330] Disable Intel invalid offsetof warning (#629) --- src/benchmark.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index da5532251b..b14bc62914 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -319,7 +319,10 @@ State::State(size_t max_iters, const std::vector& ranges, int thread_i, // demonstrated since constexpr evaluation must diagnose all undefined // behavior). However, GCC and Clang also warn about this use of offsetof, // which must be suppressed. -#ifdef __GNUC__ +#if defined(__INTEL_COMPILER) +#pragma warning push +#pragma warning(disable:1875) +#elif defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #endif @@ -328,7 +331,9 @@ State::State(size_t max_iters, const std::vector& ranges, int thread_i, static_assert(offsetof(State, error_occurred_) <= (cache_line_size - sizeof(error_occurred_)), ""); -#ifdef __GNUC__ +#if defined(__INTEL_COMPILER) +#pragma warning pop +#elif defined(__GNUC__) #pragma GCC diagnostic pop #endif } From 0c21bc369aa157c10e63f4e392fd19768ac7e05b Mon Sep 17 00:00:00 2001 From: Federico Ficarelli Date: Mon, 9 Jul 2018 12:45:10 +0200 Subject: [PATCH 061/330] Fix build with Intel compiler (#631) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Set -Wno-deprecated-declarations for Intel Intel compiler silently ignores -Wno-deprecated-declarations so warning no. 1786 must be explicitly suppressed. * Make std::int64_t → double casts explicit While std::int64_t → double is a perfectly conformant implicit conversion, Intel compiler warns about it. Make them explicit via static_cast. * Make std::int64_t → int casts explicit Intel compiler warns about emplacing an std::int64_t into an int container. Just make the conversion explicit via static_cast. * Cleanup Intel -Wno-deprecated-declarations workaround logic --- CMakeLists.txt | 6 ++++++ src/complexity.cc | 4 ++-- test/complexity_test.cc | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c7fe7a8ed4..8ddacabb6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,12 @@ else() # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + # Intel silently ignores '-Wno-deprecated-declarations', + # warning no. 1786 must be explicitly disabled. + # See #631 for rationale. + add_cxx_compiler_flag(-wd1786) + endif() # Disable deprecation warnings for release builds (when -Werror is enabled). add_cxx_compiler_flag(-Wno-deprecated RELEASE) add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) diff --git a/src/complexity.cc b/src/complexity.cc index 02fe3feb3c..aafd538df2 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -36,10 +36,10 @@ BigOFunc* FittingCurve(BigO complexity) { return [](int64_t n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](int64_t n) { return kLog2E * log(n); }; + return [](int64_t n) { return kLog2E * log(static_cast(n)); }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](int64_t n) { return kLog2E * n * log(n); }; + return [](int64_t n) { return kLog2E * n * log(static_cast(n)); }; case o1: default: return [](int64_t) { return 1.0; }; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index c732e6ed49..5f91660898 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -85,7 +85,7 @@ std::vector ConstructRandomVector(int64_t size) { std::vector v; v.reserve(static_cast(size)); for (int i = 0; i < size; ++i) { - v.push_back(std::rand() % size); + v.push_back(static_cast(std::rand() % size)); } return v; } @@ -106,7 +106,7 @@ BENCHMARK(BM_Complexity_O_N) BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) -> double { return n; }); + ->Complexity([](int64_t n) -> double { return static_cast(n); }); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -142,7 +142,7 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) { return kLog2E * n * log(n); }); + ->Complexity([](int64_t n) { return kLog2E * n * log(static_cast(n)); }); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) From 1f35fa4aa71bffb5e5672f7ca876561d6adef4fd Mon Sep 17 00:00:00 2001 From: Federico Ficarelli Date: Mon, 9 Jul 2018 13:47:16 +0200 Subject: [PATCH 062/330] Update AUTHORS and CONTRIBUTORS (#632) Adding myself to AUTHORS and CONTRIBUTORS according to guidelines. --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index f8219036d2..daea1f66f0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,6 +19,7 @@ Dominik Czarnota Eric Fiselier Eugene Zhuk Evgeny Safronov +Federico Ficarelli Felix Homann Google Inc. International Business Machines Corporation diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 1cf04db17e..2ff2f2a8fa 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -34,6 +34,7 @@ Dominik Czarnota Eric Fiselier Eugene Zhuk Evgeny Safronov +Federico Ficarelli Felix Homann Ismael Jimenez Martinez Jern-Kuan Leong From da9ec3dfca2a1266c4f35cc3d340f094cd71487a Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Mon, 9 Jul 2018 00:17:44 -0400 Subject: [PATCH 063/330] Include system load average in console and JSON reports High system load can skew benchmark results. By including system load averages in the library's output, we help users identify a potential issue in the quality of their measurements, and thus assist them in producing better (more reproducible) results. I got the idea for this from Brendan Gregg's checklist for benchmark accuracy (http://www.brendangregg.com/blog/2018-06-30/benchmarking-checklist.html). --- include/benchmark/benchmark.h | 1 + src/json_reporter.cc | 6 ++++++ src/reporter.cc | 9 +++++++++ src/sysinfo.cc | 21 ++++++++++++++++++++- test/reporter_output_test.cc | 10 ++++++++-- 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 193fffc4be..071bc332f3 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1242,6 +1242,7 @@ struct CPUInfo { double cycles_per_second; std::vector caches; bool scaling_enabled; + std::vector load_avg; static const CPUInfo& Get(); diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 611605af6b..4d1da2b3d5 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -111,6 +111,12 @@ bool JSONReporter::ReportContext(const Context& context) { } indent = std::string(4, ' '); out << indent << "],\n"; + out << indent << "\"load_avg\": ["; + for (auto it = info.load_avg.begin(); it != info.load_avg.end();) { + out << *it++; + if (it != info.load_avg.end()) out << ","; + } + out << "],\n"; #if defined(NDEBUG) const char build_type[] = "release"; diff --git a/src/reporter.cc b/src/reporter.cc index 541661a25f..74a42f6b37 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -22,6 +22,7 @@ #include #include "check.h" +#include "string_util.h" namespace benchmark { @@ -54,6 +55,14 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } } + if (!info.load_avg.empty()) { + Out << "Load Average: "; + for (auto It = info.load_avg.begin(); It != info.load_avg.end();) { + Out << StrFormat("%.2f", *It++); + if (It != info.load_avg.end()) Out << ", "; + } + Out << "\n"; + } if (info.scaling_enabled) { Out << "***WARNING*** CPU scaling is enabled, the benchmark " diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 73064b97ba..309aeb471f 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -571,6 +571,24 @@ double GetCPUCyclesPerSecond() { return static_cast(cycleclock::Now() - start_ticks); } +std::vector GetLoadAvg() { +#if defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ + defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ + defined BENCHMARK_OS_OPENBSD + constexpr int kMaxSamples = 3; + std::vector res(kMaxSamples, 0.0); + const int nelem = getloadavg(res.data(), kMaxSamples); + if (nelem < 1) { + res.clear(); + } else { + res.resize(nelem); + } + return res; +#else + return {}; +#endif +} + } // end namespace const CPUInfo& CPUInfo::Get() { @@ -582,6 +600,7 @@ CPUInfo::CPUInfo() : num_cpus(GetNumCPUs()), cycles_per_second(GetCPUCyclesPerSecond()), caches(GetCacheSizes()), - scaling_enabled(CpuScalingEnabled(num_cpus)) {} + scaling_enabled(CpuScalingEnabled(num_cpus)), + load_avg(GetLoadAvg()) {} } // end namespace benchmark diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 1662fcb8b5..64102249a3 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -28,7 +28,8 @@ static int AddContextCases() { {"\"mhz_per_cpu\": %float,$", MR_Next}, {"\"cpu_scaling_enabled\": ", MR_Next}, {"\"caches\": \\[$", MR_Next}}); - auto const& Caches = benchmark::CPUInfo::Get().caches; + auto const& Info = benchmark::CPUInfo::Get(); + auto const& Caches = Info.caches; if (!Caches.empty()) { AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}}); } @@ -45,8 +46,13 @@ static int AddContextCases() { {"\"num_sharing\": %int$", MR_Next}, {"}[,]{0,1}$", MR_Next}}); } - AddCases(TC_JSONOut, {{"],$"}}); + auto const& LoadAvg = Info.load_avg; + if (!LoadAvg.empty()) { + AddCases(TC_ConsoleErr, + {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); + } + AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); return 0; } int dummy_register = AddContextCases(); From e1150acab90b39bd7da79622c0fa340d78f09f41 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Mon, 9 Jul 2018 10:52:07 -0400 Subject: [PATCH 064/330] Add Ori Livneh to AUTHORS and CONTRIBUTORS --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index daea1f66f0..09e2e0551a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,6 +36,7 @@ Maxim Vafin MongoDB Inc. Nick Hutchinson Oleksandr Sochka +Ori Livneh Paul Redmond Radoslav Yovchev Roman Lebedev diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 2ff2f2a8fa..f727bd1d08 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -50,6 +50,7 @@ Matt Clarkson Maxim Vafin Nick Hutchinson Oleksandr Sochka +Ori Livneh Pascal Leroy Paul Redmond Pierre Phaneuf From 63e183b38945a0d67671f94bd3f74d7ffb6f5a82 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 23 Jul 2018 12:08:20 +0100 Subject: [PATCH 065/330] Add note to tools.md regarding scipy. --- docs/tools.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tools.md b/docs/tools.md index 70500bd322..cc2515f4c0 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -9,6 +9,8 @@ The program is invoked like: $ compare_bench.py [benchmark options]... ``` +Note, it relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html). + Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file. `[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes. From f965eab5083010ea7c3a8298176a837e5bff3016 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 24 Jul 2018 15:57:15 +0100 Subject: [PATCH 066/330] Memory management and reporting hooks (#625) * Introduce memory manager interface * Add memory stats to JSON reporter and a test * Add comments and switch json output test to int --- include/benchmark/benchmark.h | 43 +++++++++++++++++++++++++++++------ src/benchmark.cc | 35 ++++++++++++++++++++++++++-- src/json_reporter.cc | 6 +++++ test/memory_manager_test.cc | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 test/memory_manager_test.cc diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 193fffc4be..aef995ecdd 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -243,6 +243,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); namespace benchmark { class BenchmarkReporter; +class MemoryManager; void Initialize(int* argc, char** argv); @@ -267,12 +268,9 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter); size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, BenchmarkReporter* file_reporter); -// If this routine is called, peak memory allocation past this point in the -// benchmark is reported at the end of the benchmark report line. (It is -// computed by running the benchmark once with a single iteration and a memory -// tracer.) -// TODO(dominic) -// void MemoryUsage(); +// Register a MemoryManager instance that will be used to collect and report +// allocation measurements for benchmark runs. +void RegisterMemoryManager(MemoryManager* memory_manager); namespace internal { class Benchmark; @@ -1280,7 +1278,10 @@ class BenchmarkReporter { complexity_n(0), report_big_o(false), report_rms(false), - counters() {} + counters(), + has_memory_result(false), + allocs_per_iter(0.0), + max_bytes_used(0) {} std::string benchmark_name; std::string report_label; // Empty if not set by benchmark. @@ -1324,6 +1325,11 @@ class BenchmarkReporter { bool report_rms; UserCounters counters; + + // Memory metrics. + bool has_memory_result; + double allocs_per_iter; + int64_t max_bytes_used; }; // Construct a BenchmarkReporter with the output stream set to 'std::cout' @@ -1438,6 +1444,29 @@ class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future rel std::set user_counter_names_; }; +// If a MemoryManager is registered, it can be used to collect and report +// allocation metrics for a run of the benchmark. +class MemoryManager { + public: + struct Result { + Result() : num_allocs(0), max_bytes_used(0) {} + + // The number of allocations made in total between Start and Stop. + int64_t num_allocs; + + // The peak memory use between Start and Stop. + int64_t max_bytes_used; + }; + + virtual ~MemoryManager() {} + + // Implement this to start recording allocation information. + virtual void Start() = 0; + + // Implement this to stop recording and fill out the given Result structure. + virtual void Stop(Result* result) = 0; +}; + inline const char* GetTimeUnitString(TimeUnit unit) { switch (unit) { case kMillisecond: diff --git a/src/benchmark.cc b/src/benchmark.cc index b14bc62914..0f2e198a8f 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -105,6 +105,8 @@ namespace benchmark { namespace { static const size_t kMaxIterations = 1000000000; + +static MemoryManager* memory_manager = nullptr; } // end namespace namespace internal { @@ -115,7 +117,8 @@ namespace { BenchmarkReporter::Run CreateRunReport( const benchmark::internal::Benchmark::Instance& b, - const internal::ThreadManager::Result& results, double seconds) { + const internal::ThreadManager::Result& results, size_t memory_iterations, + const MemoryManager::Result& memory_result, double seconds) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -150,6 +153,16 @@ BenchmarkReporter::Run CreateRunReport( report.complexity_lambda = b.complexity_lambda; report.statistics = b.statistics; report.counters = results.counters; + + if (memory_iterations > 0) { + report.has_memory_result = true; + report.allocs_per_iter = + memory_iterations ? static_cast(memory_result.num_allocs) / + memory_iterations + : 0; + report.max_bytes_used = memory_result.max_bytes_used; + } + internal::Finish(&report.counters, results.iterations, seconds, b.threads); } return report; @@ -249,7 +262,23 @@ std::vector RunBenchmark( // clang-format on if (should_report) { - BenchmarkReporter::Run report = CreateRunReport(b, results, seconds); + MemoryManager::Result memory_result; + size_t memory_iterations = 0; + if (memory_manager != nullptr) { + // Only run a few iterations to reduce the impact of one-time + // allocations in benchmarks that are not properly managed. + memory_iterations = std::min(16, iters); + memory_manager->Start(); + manager.reset(new internal::ThreadManager(1)); + RunInThread(&b, memory_iterations, 0, manager.get()); + manager->WaitForAllThreads(); + manager.reset(); + + memory_manager->Stop(&memory_result); + } + + BenchmarkReporter::Run report = CreateRunReport( + b, results, memory_iterations, memory_result, seconds); if (!report.error_occurred && b.complexity != oNone) complexity_reports->push_back(report); reports.push_back(report); @@ -555,6 +584,8 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, return benchmarks.size(); } +void RegisterMemoryManager(MemoryManager* manager) { memory_manager = manager; } + namespace internal { void PrintUsageAndExit() { diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 611605af6b..6d0706f1e4 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -186,6 +186,12 @@ void JSONReporter::PrintRunData(Run const& run) { for (auto& c : run.counters) { out << ",\n" << indent << FormatKV(c.first, c.second); } + + if (run.has_memory_result) { + out << ",\n" << indent << FormatKV("allocs_per_iter", run.allocs_per_iter); + out << ",\n" << indent << FormatKV("max_bytes_used", run.max_bytes_used); + } + if (!run.report_label.empty()) { out << ",\n" << indent << FormatKV("label", run.report_label); } diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc new file mode 100644 index 0000000000..da735938ae --- /dev/null +++ b/test/memory_manager_test.cc @@ -0,0 +1,40 @@ +#include + +#include "../src/check.h" +#include "benchmark/benchmark.h" +#include "output_test.h" + +class TestMemoryManager : public benchmark::MemoryManager { + void Start() {} + void Stop(Result* result) { + result->num_allocs = 42; + result->max_bytes_used = 42000; + } +}; + +void BM_empty(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(state.iterations()); + } +} +BENCHMARK(BM_empty); + +ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"allocs_per_iter\": %float,$", MR_Next}, + {"\"max_bytes_used\": 42000$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); + + +int main(int argc, char *argv[]) { + std::unique_ptr mm(new TestMemoryManager()); + + benchmark::RegisterMemoryManager(mm.get()); + RunOutputTests(argc, argv); + benchmark::RegisterMemoryManager(nullptr); +} From d939634b8ce7e0741a79c1c1f22205fae54b375d Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 26 Jul 2018 14:29:33 +0100 Subject: [PATCH 067/330] README improvements (#648) * Clarifications and cleaning of the core documentation. --- README.md | 110 +++++++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 80e69f6e10..db3badf774 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,9 @@ A library to support the benchmarking of functions, similar to unit-tests. -Discussion group: https://groups.google.com/d/forum/benchmark-discuss +[Discussion group](https://groups.google.com/d/forum/benchmark-discuss) -IRC channel: https://freenode.net #googlebenchmark - -[Known issues and common problems](#known-issues) +IRC channel: [freenode](https://freenode.net) #googlebenchmark [Additional Tooling Documentation](docs/tools.md) @@ -47,11 +45,10 @@ to `CMAKE_ARGS`. For Ubuntu and Debian Based System -First make sure you have git and cmake installed (If not please install it) +First make sure you have git and cmake installed (If not please install them) ``` -sudo apt-get install git -sudo apt-get install cmake +sudo apt-get install git cmake ``` Now, let's clone the repository and build it @@ -59,22 +56,20 @@ Now, let's clone the repository and build it ``` git clone https://github.com/google/benchmark.git cd benchmark -git clone https://github.com/google/googletest.git +# If you want to build tests and don't use BENCHMARK_DOWNLOAD_DEPENDENCIES, then +# git clone https://github.com/google/googletest.git mkdir build cd build cmake .. -DCMAKE_BUILD_TYPE=RELEASE make ``` -We need to install the library globally now +If you need to install the library globally ``` sudo make install ``` -Now you have google/benchmark installed in your machine -Note: Don't forget to link to pthread library while building - ## Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; @@ -87,15 +82,16 @@ to use, test, and provide feedback on the new features are encouraged to try this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. -## Prerequisite knowledge - -Before attempting to understand this framework one should ideally have some familiarity with the structure and format of the Google Test framework, upon which it is based. Documentation for Google Test, including a "Getting Started" (primer) guide, is available here: -https://github.com/google/googletest/blob/master/googletest/docs/primer.md +## Further knowledge +It may help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) +as some of the structural aspects of the APIs are similar. ## Example usage ### Basic usage -Define a function that executes the code to be measured. +Define a function that executes the code to be measured, register it as a +benchmark function using the `BENCHMARK` macro, and ensure an appropriate `main` +function is available: ```c++ #include @@ -123,7 +119,23 @@ Don't forget to inform your linker to add benchmark library e.g. through `BENCHMARK_MAIN();` at the end of the source file and link against `-lbenchmark_main` to get the same default behavior. -The benchmark library will reporting the timing for the code within the `for(...)` loop. +The benchmark library will measure and report the timing for code within the +`for(...)` loop. + +#### Platform-specific libraries +When the library is built using GCC it is necessary to link with the pthread +library due to how GCC implements `std::thread`. Failing to link to pthread will +lead to runtime exceptions (unless you're using libc++), not linker errors. See +[issue #67](https://github.com/google/benchmark/issues/67) for more details. You +can link to pthread by adding `-pthread` to your linker command. Note, you can +also use `-lpthread`, but there are potential issues with ordering of command +line parameters if you use that. + +If you're running benchmarks on Windows, the shlwapi library (`-lshlwapi`) is +also required. + +If you're running benchmarks on solaris, you'll want the kstat library linked in +too (`-lkstat`). ### Passing arguments Sometimes a family of benchmarks can be implemented with just one routine that @@ -522,15 +534,7 @@ order to manually set the time unit, you can specify it manually: BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); ``` -## Controlling number of iterations -In all cases, the number of iterations for which the benchmark is run is -governed by the amount of time the benchmark takes. Concretely, the number of -iterations is at least one, not more than 1e9, until CPU time is greater than -the minimum time, or the wallclock time is 5x minimum time. The minimum time is -set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on -the registered benchmark object. - -## Reporting the mean, median and standard deviation by repeated benchmarks +### Reporting the mean, median and standard deviation by repeated benchmarks By default each benchmark is run once and that single result is reported. However benchmarks are often noisy and a single result may not be representative of the overall behavior. For this reason it's possible to repeatedly rerun the @@ -810,8 +814,29 @@ BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32k 1834 ns 1837 ns 357143 ``` +## Runtime and reporting considerations +When the benchmark binary is executed, each benchmark function is run serially. +The number of iterations to run is determined dynamically by running the +benchmark a few times and measuring the time taken and ensuring that the +ultimate result will be statistically stable. As such, faster benchmark +functions will be run for more iterations than slower benchmark functions, and +the number of iterations is thus reported. -## Output Formats +In all cases, the number of iterations for which the benchmark is run is +governed by the amount of time the benchmark takes. Concretely, the number of +iterations is at least one, not more than 1e9, until CPU time is greater than +the minimum time, or the wallclock time is 5x minimum time. The minimum time is +set per benchmark by calling `MinTime` on the registered benchmark object. + +Average timings are then reported over the iterations run. If multiple +repetitions are requested using the `--benchmark_repetitions` command-line +option, or at registration time, the benchmark function will be run several +times and statistical results across these repetitions will also be reported. + +As well as the per-benchmark entries, a preamble in the report will include +information about the machine on which the benchmarks are run. + +### Output Formats The library supports multiple output formats. Use the `--benchmark_format=` flag to set the format type. `console` is the default format. @@ -879,14 +904,15 @@ name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label "BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, ``` -## Output Files +### Output Files The library supports writing the output of the benchmark to a file specified by `--benchmark_out=`. The format of the output can be specified using `--benchmark_out_format={json|console|csv}`. Specifying `--benchmark_out` does not suppress the console output. ## Debug vs Release -By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use: +By default, benchmark builds as a debug library. You will see a warning in the +output when this is the case. To build it as a release library instead, use: ``` cmake -DCMAKE_BUILD_TYPE=Release @@ -898,16 +924,11 @@ To enable link-time optimisation, use cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true ``` -If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake cache variables, if autodetection fails. -If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. - -## Linking against the library +If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake +cache variables, if autodetection fails. -When the library is built using GCC it is necessary to link with `-pthread`, -due to how GCC implements `std::thread`. - -For GCC 4.x failing to link to pthreads will lead to runtime exceptions, not linker errors. -See [issue #67](https://github.com/google/benchmark/issues/67) for more details. +If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, +`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. ## Compiler Support @@ -937,14 +958,3 @@ sudo cpupower frequency-set --governor performance ./mybench sudo cpupower frequency-set --governor powersave ``` - -# Known Issues - -### Windows with CMake - -* Users must manually link `shlwapi.lib`. Failure to do so may result -in unresolved symbols. - -### Solaris - -* Users must explicitly link with kstat library (-lkstat compilation flag). From f85304e4e3a0e4e1bf15b91720df4a19e90b589f Mon Sep 17 00:00:00 2001 From: Kirill Bobyrev Date: Wed, 8 Aug 2018 15:39:57 +0200 Subject: [PATCH 068/330] Remove redundant default which causes failures (#649) * Remove redundant default which causes failures * Fix old GCC warnings caused by poor analysis * Use __builtin_unreachable * Use BENCHMARK_UNREACHABLE() * Pull __has_builtin to benchmark.h too * Also move compiler identification macro to main header * Move custom compiler identification macro back --- include/benchmark/benchmark.h | 16 ++++++++++++++-- src/internal_macros.h | 11 ----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index aef995ecdd..7434a8a533 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -241,6 +241,18 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #endif +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if defined(__GNUC__) || __has_builtin(__builtin_unreachable) + #define BENCHMARK_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) + #define BENCHMARK_UNREACHABLE() __assume(false) +#else + #define BENCHMARK_UNREACHABLE() ((void)0) +#endif + namespace benchmark { class BenchmarkReporter; class MemoryManager; @@ -1474,9 +1486,9 @@ inline const char* GetTimeUnitString(TimeUnit unit) { case kMicrosecond: return "us"; case kNanosecond: - default: return "ns"; } + BENCHMARK_UNREACHABLE(); } inline double GetTimeUnitMultiplier(TimeUnit unit) { @@ -1486,9 +1498,9 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { case kMicrosecond: return 1e6; case kNanosecond: - default: return 1e9; } + BENCHMARK_UNREACHABLE(); } } // namespace benchmark diff --git a/src/internal_macros.h b/src/internal_macros.h index b7e9203ff6..32089e69f6 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -11,9 +11,6 @@ #ifndef __has_feature #define __has_feature(x) 0 #endif -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif #if defined(__clang__) #if !defined(COMPILER_CLANG) @@ -87,14 +84,6 @@ #define BENCHMARK_MAYBE_UNUSED #endif -#if defined(COMPILER_GCC) || __has_builtin(__builtin_unreachable) - #define BENCHMARK_UNREACHABLE() __builtin_unreachable() -#elif defined(COMPILER_MSVC) - #define BENCHMARK_UNREACHABLE() __assume(false) -#else - #define BENCHMARK_UNREACHABLE() ((void)0) -#endif - // clang-format on #endif // BENCHMARK_INTERNAL_MACROS_H_ From 94c4d6d5c65b8f0a5f9963db7d93e9568f8aa3db Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 13 Aug 2018 17:42:35 +0300 Subject: [PATCH 069/330] [Tools] Drop compare_bench.py, compare.py is to be used, add U-test docs. (#645) As discussed in IRC, time to deduplicate. --- README.md | 4 ++ docs/tools.md | 96 +++++++++--------------------------------- tools/compare_bench.py | 67 ----------------------------- 3 files changed, 25 insertions(+), 142 deletions(-) delete mode 100755 tools/compare_bench.py diff --git a/README.md b/README.md index db3badf774..770e46658d 100644 --- a/README.md +++ b/README.md @@ -910,6 +910,10 @@ by `--benchmark_out=`. The format of the output can be specified using `--benchmark_out_format={json|console|csv}`. Specifying `--benchmark_out` does not suppress the console output. +## Result comparison + +It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) + ## Debug vs Release By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use: diff --git a/docs/tools.md b/docs/tools.md index cc2515f4c0..73ba27852e 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -1,86 +1,16 @@ # Benchmark Tools -## compare_bench.py - -The `compare_bench.py` utility which can be used to compare the result of benchmarks. -The program is invoked like: - -``` bash -$ compare_bench.py [benchmark options]... -``` - -Note, it relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html). - -Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file. - -`[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes. - -The sample output using the JSON test files under `Inputs/` gives: - -``` bash -$ ./compare_bench.py ./gbench/Inputs/test1_run1.json ./gbench/Inputs/test1_run2.json -Comparing ./gbench/Inputs/test1_run1.json to ./gbench/Inputs/test1_run2.json -Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------- -BM_SameTimes +0.0000 +0.0000 10 10 10 10 -BM_2xFaster -0.5000 -0.5000 50 25 50 25 -BM_2xSlower +1.0000 +1.0000 50 100 50 100 -BM_1PercentFaster -0.0100 -0.0100 100 99 100 99 -BM_1PercentSlower +0.0100 +0.0100 100 101 100 101 -BM_10PercentFaster -0.1000 -0.1000 100 90 100 90 -BM_10PercentSlower +0.1000 +0.1000 100 110 100 110 -BM_100xSlower +99.0000 +99.0000 100 10000 100 10000 -BM_100xFaster -0.9900 -0.9900 10000 100 10000 100 -BM_10PercentCPUToTime +0.1000 -0.1000 100 110 100 90 -BM_ThirdFaster -0.3333 -0.3334 100 67 100 67 -BM_BadTimeUnit -0.9000 +0.2000 0 0 0 1 -``` - -As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. - -When a benchmark executable is run, the raw output from the benchmark is printed in real time to stdout. The sample output using `benchmark/basic_test` for both arguments looks like: +## compare.py -``` -./compare_bench.py test/basic_test test/basic_test --benchmark_filter=BM_empty.* -RUNNING: test/basic_test --benchmark_filter=BM_empty.* --benchmark_out=/tmp/tmpN7LF3a -Run on (8 X 4000 MHz CPU s) -2017-11-07 23:28:36 ---------------------------------------------------------------------- -Benchmark Time CPU Iterations ---------------------------------------------------------------------- -BM_empty 4 ns 4 ns 170178757 -BM_empty/threads:8 1 ns 7 ns 103868920 -BM_empty_stop_start 0 ns 0 ns 1000000000 -BM_empty_stop_start/threads:8 0 ns 0 ns 1403031720 -RUNNING: /test/basic_test --benchmark_filter=BM_empty.* --benchmark_out=/tmp/tmplvrIp8 -Run on (8 X 4000 MHz CPU s) -2017-11-07 23:28:38 ---------------------------------------------------------------------- -Benchmark Time CPU Iterations ---------------------------------------------------------------------- -BM_empty 4 ns 4 ns 169534855 -BM_empty/threads:8 1 ns 7 ns 104188776 -BM_empty_stop_start 0 ns 0 ns 1000000000 -BM_empty_stop_start/threads:8 0 ns 0 ns 1404159424 -Comparing ../build/test/basic_test to ../build/test/basic_test -Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- -BM_empty -0.0048 -0.0049 4 4 4 4 -BM_empty/threads:8 -0.0123 -0.0054 1 1 7 7 -BM_empty_stop_start -0.0000 -0.0000 0 0 0 0 -BM_empty_stop_start/threads:8 -0.0029 +0.0001 0 0 0 0 +The `compare.py` can be used to compare the result of benchmarks. -``` +**NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html). -As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. -Obviously this example doesn't give any useful output, but it's intended to show the output format when 'compare_bench.py' needs to run benchmarks. +### Modes of operation -## compare.py - -The `compare.py` can be used to compare the result of benchmarks. There are three modes of operation: -1. Just compare two benchmarks, what `compare_bench.py` did. +1. Just compare two benchmarks The program is invoked like: ``` bash @@ -242,3 +172,19 @@ Benchmark Time CPU Time Old ``` This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one. As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. + +### U test + +If there is a sufficient repetition count of the benchmarks, the tool can do +a [U Test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), of the +null hypothesis that it is equally likely that a randomly selected value from +one sample will be less than or greater than a randomly selected value from a +second sample. + +If the calculated p-value is below this value is lower than the significance +level alpha, then the result is said to be statistically significant and the +null hypothesis is rejected. Which in other words means that the two benchmarks +aren't identical. + +**WARNING**: requires **LARGE** (no less than 9) number of repetitions to be +meaningful! diff --git a/tools/compare_bench.py b/tools/compare_bench.py deleted file mode 100755 index 7bbf0d0157..0000000000 --- a/tools/compare_bench.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -""" -compare_bench.py - Compare two benchmarks or their results and report the - difference. -""" -import argparse -from argparse import ArgumentParser -import sys -import gbench -from gbench import util, report -from gbench.util import * - -def check_inputs(in1, in2, flags): - """ - Perform checking on the user provided inputs and diagnose any abnormalities - """ - in1_kind, in1_err = classify_input_file(in1) - in2_kind, in2_err = classify_input_file(in2) - output_file = find_benchmark_flag('--benchmark_out=', flags) - output_type = find_benchmark_flag('--benchmark_out_format=', flags) - if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: - print(("WARNING: '--benchmark_out=%s' will be passed to both " - "benchmarks causing it to be overwritten") % output_file) - if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: - print("WARNING: passing --benchmark flags has no effect since both " - "inputs are JSON") - if output_type is not None and output_type != 'json': - print(("ERROR: passing '--benchmark_out_format=%s' to 'compare_bench.py`" - " is not supported.") % output_type) - sys.exit(1) - - -def main(): - parser = ArgumentParser( - description='compare the results of two benchmarks') - parser.add_argument( - 'test1', metavar='test1', type=str, nargs=1, - help='A benchmark executable or JSON output file') - parser.add_argument( - 'test2', metavar='test2', type=str, nargs=1, - help='A benchmark executable or JSON output file') - parser.add_argument( - 'benchmark_options', metavar='benchmark_options', nargs=argparse.REMAINDER, - help='Arguments to pass when running benchmark executables' - ) - args, unknown_args = parser.parse_known_args() - # Parse the command line flags - test1 = args.test1[0] - test2 = args.test2[0] - if unknown_args: - # should never happen - print("Unrecognized positional argument arguments: '%s'" - % unknown_args) - exit(1) - benchmark_options = args.benchmark_options - check_inputs(test1, test2, benchmark_options) - # Run the benchmarks and report the results - json1 = gbench.util.run_or_load_benchmark(test1, benchmark_options) - json2 = gbench.util.run_or_load_benchmark(test2, benchmark_options) - output_lines = gbench.report.generate_difference_report(json1, json2) - print('Comparing %s to %s' % (test1, test2)) - for ln in output_lines: - print(ln) - - -if __name__ == '__main__': - main() From af441fc1143e33e539ceec4df67c2d95ac2bf5f8 Mon Sep 17 00:00:00 2001 From: BaaMeow <38274252+BaaMeow@users.noreply.github.com> Date: Thu, 16 Aug 2018 12:47:09 -0400 Subject: [PATCH 070/330] properly escape json names (#652) --- src/json_reporter.cc | 7 ++++++- test/reporter_output_test.cc | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 6d0706f1e4..127a96a06c 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -78,7 +78,12 @@ bool JSONReporter::ReportContext(const Context& context) { out << indent << FormatKV("date", walltime_value) << ",\n"; if (Context::executable_name) { - out << indent << FormatKV("executable", Context::executable_name) << ",\n"; + // windows uses backslash for its path separator, + // which must be escaped in JSON otherwise it blows up conforming JSON + // decoders + std::string executable_name = Context::executable_name; + ReplaceAll(&executable_name, "\\", "\\\\"); + out << indent << FormatKV("executable", executable_name) << ",\n"; } CPUInfo const& info = context.cpu_info; diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 1662fcb8b5..80f3e78602 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -23,7 +23,8 @@ static int AddContextCases() { {{"^\\{", MR_Default}, {"\"context\":", MR_Next}, {"\"date\": \"", MR_Next}, - {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next}, + {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",", + MR_Next}, {"\"num_cpus\": %int,$", MR_Next}, {"\"mhz_per_cpu\": %float,$", MR_Next}, {"\"cpu_scaling_enabled\": ", MR_Next}, From ede90ba6c8091e457592f2d78c49d7c91a40a06c Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Tue, 28 Aug 2018 16:10:14 +0200 Subject: [PATCH 071/330] Make tests pass on 1-core VMs (#653) found while working on reproducible builds for openSUSE To reproduce there osc checkout openSUSE:Factory/benchmark && cd $_ osc build -j1 --vm-type=kvm --- test/reporter_output_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 80f3e78602..99fe2c20c2 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -17,7 +17,7 @@ static int AddContextCases() { { {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, - {"Run on \\(%int X %float MHz CPU s\\)", MR_Next}, + {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); AddCases(TC_JSONOut, {{"^\\{", MR_Default}, From 9a179cb93faf76bbbff8410849ecb2eb3cf65024 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 28 Aug 2018 17:19:25 +0300 Subject: [PATCH 072/330] [NFC] Prefix "report(_)?mode" with Aggregation. (#656) This only specifically represents handling of reporting of aggregates. Not of anything else. Making it more specific makes the name less generic. This is an issue because i want to add "iteration report mode", so the naming would be conflicting. --- include/benchmark/benchmark.h | 10 +++++----- src/benchmark.cc | 4 ++-- src/benchmark_api_internal.h | 2 +- src/benchmark_register.cc | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 7434a8a533..87b10d1f90 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -424,14 +424,14 @@ namespace internal { class ThreadTimer; class ThreadManager; -enum ReportMode +enum AggregationReportMode #if defined(BENCHMARK_HAS_CXX11) : unsigned #else #endif -{ RM_Unspecified, // The mode has not been manually specified - RM_Default, // The mode is user-specified as default. - RM_ReportAggregatesOnly }; +{ ARM_Unspecified, // The mode has not been manually specified + ARM_Default, // The mode is user-specified as default. + ARM_ReportAggregatesOnly }; } // namespace internal // State is passed to a running Benchmark and contains state for the @@ -912,7 +912,7 @@ class Benchmark { friend class BenchmarkFamilies; std::string name_; - ReportMode report_mode_; + AggregationReportMode aggregation_report_mode_; std::vector arg_names_; // Args for all benchmark runs std::vector > args_; // Args for all benchmark runs TimeUnit time_unit_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 0f2e198a8f..f9c8686517 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -206,9 +206,9 @@ std::vector RunBenchmark( b.repetitions != 0 ? b.repetitions : FLAGS_benchmark_repetitions; const bool report_aggregates_only = repeats != 1 && - (b.report_mode == internal::RM_Unspecified + (b.aggregation_report_mode == internal::ARM_Unspecified ? FLAGS_benchmark_report_aggregates_only - : b.report_mode == internal::RM_ReportAggregatesOnly); + : b.aggregation_report_mode == internal::ARM_ReportAggregatesOnly); for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { for (;;) { // Try benchmark diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index dd7a3ffe8c..0bf0005e84 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -16,7 +16,7 @@ namespace internal { struct Benchmark::Instance { std::string name; Benchmark* benchmark; - ReportMode report_mode; + AggregationReportMode aggregation_report_mode; std::vector arg; TimeUnit time_unit; int range_multiplier; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 26a89721c7..8ee30adf0f 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -155,7 +155,7 @@ bool BenchmarkFamilies::FindBenchmarks( Benchmark::Instance instance; instance.name = family->name_; instance.benchmark = family.get(); - instance.report_mode = family->report_mode_; + instance.aggregation_report_mode = family->aggregation_report_mode_; instance.arg = args; instance.time_unit = family->time_unit_; instance.range_multiplier = family->range_multiplier_; @@ -236,7 +236,7 @@ bool FindBenchmarksInternal(const std::string& re, Benchmark::Benchmark(const char* name) : name_(name), - report_mode_(RM_Unspecified), + aggregation_report_mode_(ARM_Unspecified), time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0), @@ -369,7 +369,7 @@ Benchmark* Benchmark::Repetitions(int n) { } Benchmark* Benchmark::ReportAggregatesOnly(bool value) { - report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default; + aggregation_report_mode_ = value ? ARM_ReportAggregatesOnly : ARM_Default; return this; } From 8688c5c4cfa1527ceca2136b2a738d9712a01890 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 28 Aug 2018 18:11:36 +0300 Subject: [PATCH 073/330] Track 'type' of the run - is it an actual measurement, or an aggregate. (#658) This is *only* exposed in the JSON. Not in CSV, which is deprecated. This *only* supposed to track these two states. An additional field could later track which aggregate this is, specifically (statistic name, rms, bigo, ...) The motivation is that we already have ReportAggregatesOnly, but it affects the entire reports, both the display, and the reporters (json files), which isn't ideal. It would be very useful to have a 'display aggregates only' option, both in the library's console reporter, and the python tooling, This will be especially needed for the 'store separate iterations'. --- include/benchmark/benchmark.h | 6 +++- src/complexity.cc | 2 ++ src/json_reporter.cc | 9 +++++ src/statistics.cc | 1 + test/complexity_test.cc | 2 ++ test/memory_manager_test.cc | 1 + test/reporter_output_test.cc | 58 +++++++++++++++++++++++++----- test/user_counters_tabular_test.cc | 5 +++ test/user_counters_test.cc | 10 ++++++ 9 files changed, 84 insertions(+), 10 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 87b10d1f90..efbbd72166 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1276,8 +1276,11 @@ class BenchmarkReporter { }; struct Run { + enum RunType { RT_Iteration, RT_Aggregate }; + Run() - : error_occurred(false), + : run_type(RT_Iteration), + error_occurred(false), iterations(1), time_unit(kNanosecond), real_accumulated_time(0), @@ -1296,6 +1299,7 @@ class BenchmarkReporter { max_bytes_used(0) {} std::string benchmark_name; + RunType run_type; // is this a measurement, or an aggregate? std::string report_label; // Empty if not set by benchmark. bool error_occurred; std::string error_message; diff --git a/src/complexity.cc b/src/complexity.cc index aafd538df2..8d1392a3cc 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -187,6 +187,7 @@ std::vector ComputeBigO( // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; + big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; big_o.benchmark_name = benchmark_name + "_BigO"; big_o.iterations = 0; big_o.real_accumulated_time = result_real.coef; @@ -204,6 +205,7 @@ std::vector ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; big_o.report_label = reports[0].report_label; + rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.benchmark_name = benchmark_name + "_RMS"; rms.report_label = big_o.report_label; rms.iterations = 0; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 127a96a06c..54e3af38d9 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -160,6 +160,15 @@ void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name) << ",\n"; + out << indent << FormatKV("run_type", [&run]() -> const char* { + switch (run.run_type) { + case BenchmarkReporter::Run::RT_Iteration: + return "iteration"; + case BenchmarkReporter::Run::RT_Aggregate: + return "aggregate"; + } + BENCHMARK_UNREACHABLE(); + }()) << ",\n"; if (run.error_occurred) { out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; out << indent << FormatKV("error_message", run.error_message) << ",\n"; diff --git a/src/statistics.cc b/src/statistics.cc index 612dda2d1a..2795011afe 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -150,6 +150,7 @@ std::vector ComputeStats( for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; + data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_; data.report_label = report_label; data.iterations = run_iterations; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 5f91660898..d09a2f8d65 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -25,12 +25,14 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"rms\": %float$", MR_Next}, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index da735938ae..4e71b649a4 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -21,6 +21,7 @@ BENCHMARK(BM_empty); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 99fe2c20c2..a98fc42a45 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -65,6 +65,7 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -86,6 +87,7 @@ BENCHMARK(BM_bytes_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -108,6 +110,7 @@ BENCHMARK(BM_items_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -129,6 +132,7 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -150,6 +154,7 @@ void BM_error(benchmark::State& state) { BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, {"\"error_message\": \"message\",$", MR_Next}}); @@ -166,7 +171,8 @@ void BM_no_arg_name(benchmark::State& state) { } BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // @@ -179,7 +185,8 @@ void BM_arg_name(benchmark::State& state) { } BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); // ========================================================================= // @@ -193,7 +200,8 @@ void BM_arg_names(benchmark::State& state) { BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); // ========================================================================= // @@ -228,10 +236,15 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"}, {"^BM_Repeat/repeats:2_median %console_report$"}, {"^BM_Repeat/repeats:2_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}}); + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, @@ -246,11 +259,17 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3_median %console_report$"}, {"^BM_Repeat/repeats:3_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -267,12 +286,19 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"}, {"^BM_Repeat/repeats:4_median %console_report$"}, {"^BM_Repeat/repeats:4_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}}); + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -289,7 +315,8 @@ void BM_RepeatOnce(benchmark::State& state) { } BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); // Test that non-aggregate data is not reported @@ -305,8 +332,11 @@ ADD_CASES(TC_ConsoleOut, {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, @@ -327,10 +357,13 @@ ADD_CASES(TC_ConsoleOut, {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"time_unit\": \"us\",?$"}}); ADD_CASES(TC_CSVOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, @@ -365,12 +398,19 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, {"^BM_UserStats/repeats:3_stddev %console_report$"}, {"^BM_UserStats/repeats:3_ %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_median\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, - {"\"name\": \"BM_UserStats/repeats:3_\",$"}}); + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_UserStats/repeats:3_\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"}, diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 4f126b6d97..88cb3d23ef 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -70,6 +70,7 @@ void BM_Counters_Tabular(benchmark::State& state) { } BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -114,6 +115,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) { } BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -158,6 +160,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { } BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -190,6 +193,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { } BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -226,6 +230,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { } BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 7f7ccb9f77..2c7a7b11b7 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -70,6 +71,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -110,6 +112,7 @@ ADD_CASES( TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -142,6 +145,7 @@ BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -175,6 +179,7 @@ BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " "%console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -210,6 +215,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -243,6 +249,7 @@ BENCHMARK(BM_Counters_IterationInvariant); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -281,6 +288,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -317,6 +325,7 @@ BENCHMARK(BM_Counters_AvgIterations); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -352,6 +361,7 @@ BENCHMARK(BM_Counters_kAvgIterationsRate); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, From a0018c39310da35048083e4759774ba0e6cee468 Mon Sep 17 00:00:00 2001 From: "Michael \"Croydon\" Keck" Date: Wed, 29 Aug 2018 13:51:20 +0200 Subject: [PATCH 074/330] Ignore 32 bit build option when using MSVC (#638) --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ddacabb6e..8631199211 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,11 @@ option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF) option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library." OFF) -option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) +if(NOT MSVC) + option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) +else() + set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) +endif() option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON) # Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which From d9cab612e40017af10bddaa5b60c7067032a9e1c Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 29 Aug 2018 14:58:54 +0300 Subject: [PATCH 075/330] [NFC] s/console_reporter/display_reporter/ (#663) There are two destinations: * display (console, terminal) and * file. And each of the destinations can be poplulated with one of the reporters: * console - human-friendly table-like display * json * csv (deprecated) So using the name console_reporter is confusing. Is it talking about the console reporter in the sense of table-like reporter, or in the sense of display destination? --- include/benchmark/benchmark.h | 6 +++--- src/benchmark.cc | 36 +++++++++++++++++------------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index efbbd72166..c21ccf741e 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -268,7 +268,7 @@ bool ReportUnrecognizedArguments(int argc, char** argv); // of each matching benchmark. Otherwise run each matching benchmark and // report the results. // -// The second and third overload use the specified 'console_reporter' and +// The second and third overload use the specified 'display_reporter' and // 'file_reporter' respectively. 'file_reporter' will write to the file // specified // by '--benchmark_output'. If '--benchmark_output' is not given the @@ -276,8 +276,8 @@ bool ReportUnrecognizedArguments(int argc, char** argv); // // RETURNS: The number of matching benchmarks. size_t RunSpecifiedBenchmarks(); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); // Register a MemoryManager instance that will be used to collect and report diff --git a/src/benchmark.cc b/src/benchmark.cc index f9c8686517..81a520cdf2 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -424,10 +424,10 @@ namespace internal { namespace { void RunBenchmarks(const std::vector& benchmarks, - BenchmarkReporter* console_reporter, + BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { // Note the file_reporter can be null. - CHECK(console_reporter != nullptr); + CHECK(display_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. bool has_repetitions = FLAGS_benchmark_repetitions > 1; @@ -458,22 +458,22 @@ void RunBenchmarks(const std::vector& benchmarks, std::flush(reporter->GetErrorStream()); }; - if (console_reporter->ReportContext(context) && + if (display_reporter->ReportContext(context) && (!file_reporter || file_reporter->ReportContext(context))) { - flushStreams(console_reporter); + flushStreams(display_reporter); flushStreams(file_reporter); for (const auto& benchmark : benchmarks) { std::vector reports = RunBenchmark(benchmark, &complexity_reports); - console_reporter->ReportRuns(reports); + display_reporter->ReportRuns(reports); if (file_reporter) file_reporter->ReportRuns(reports); - flushStreams(console_reporter); + flushStreams(display_reporter); flushStreams(file_reporter); } } - console_reporter->Finalize(); + display_reporter->Finalize(); if (file_reporter) file_reporter->Finalize(); - flushStreams(console_reporter); + flushStreams(display_reporter); flushStreams(file_reporter); } @@ -523,11 +523,11 @@ size_t RunSpecifiedBenchmarks() { return RunSpecifiedBenchmarks(nullptr, nullptr); } -size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter) { - return RunSpecifiedBenchmarks(console_reporter, nullptr); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) { + return RunSpecifiedBenchmarks(display_reporter, nullptr); } -size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") @@ -535,15 +535,15 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, // Setup the reporters std::ofstream output_file; - std::unique_ptr default_console_reporter; + std::unique_ptr default_display_reporter; std::unique_ptr default_file_reporter; - if (!console_reporter) { - default_console_reporter = internal::CreateReporter( + if (!display_reporter) { + default_display_reporter = internal::CreateReporter( FLAGS_benchmark_format, internal::GetOutputOptions()); - console_reporter = default_console_reporter.get(); + display_reporter = default_display_reporter.get(); } - auto& Out = console_reporter->GetOutputStream(); - auto& Err = console_reporter->GetErrorStream(); + auto& Out = display_reporter->GetOutputStream(); + auto& Err = display_reporter->GetErrorStream(); std::string const& fname = FLAGS_benchmark_out; if (fname.empty() && file_reporter) { @@ -578,7 +578,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, if (FLAGS_benchmark_list_tests) { for (auto const& benchmark : benchmarks) Out << benchmark.name << "\n"; } else { - internal::RunBenchmarks(benchmarks, console_reporter, file_reporter); + internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } return benchmarks.size(); From caa2fcb19c83e233bdcaf56d299b9951b8414066 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 29 Aug 2018 21:11:06 +0300 Subject: [PATCH 076/330] Counter(): add 'one thousand' param. (#657) * Counter(): add 'one thousand' param. Needed for https://github.com/google/benchmark/pull/654 Custom user counters are quite custom. It is not guaranteed that the user *always* expects for these to have 1k == 1000. If the counter represents bytes/memory/etc, 1k should be 1024. Some bikeshedding points: 1. Is this sufficient, or do we really want to go full on into custom types with names? I think just the '1000' is sufficient for now. 2. Should there be a helper benchmark::Counter::Counter{1000,1024}() static 'constructor' functions, since these two, by far, will be the most used? 3. In the future, we should be somehow encoding this info into JSON. * Counter(): use std::pair<> to represent 'one thousand' * Counter(): just use a new enum with two values 1000 vs 1024. Simpler is better. If someone comes up with a real reason to need something more advanced, it can be added later on. * Counter: just store the 1000 or 1024 in the One_K values directly * Counter: s/One_K/OneK/ --- README.md | 12 ++- include/benchmark/benchmark.h | 11 +- src/console_reporter.cc | 2 +- src/statistics.cc | 3 +- test/CMakeLists.txt | 3 + test/user_counters_thousands_test.cc | 153 +++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 6 deletions(-) create mode 100644 test/user_counters_thousands_test.cc diff --git a/README.md b/README.md index 770e46658d..2f2d78f6ef 100644 --- a/README.md +++ b/README.md @@ -657,9 +657,12 @@ In multithreaded benchmarks, each counter is set on the calling thread only. When the benchmark finishes, the counters from each thread will be summed; the resulting sum is the value which will be shown for the benchmark. -The `Counter` constructor accepts two parameters: the value as a `double` -and a bit flag which allows you to show counters as rates and/or as -per-thread averages: +The `Counter` constructor accepts three parameters: the value as a `double` +; a bit flag which allows you to show counters as rates, and/or as per-thread +iteration, and/or as per-thread averages, and/or iteration invariants; +and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, +`benchmark::Counter::OneK::kIs1000`), or 1024 +(`benchmark::Counter::OneK::kIs1024`)? ```c++ // sets a simple counter @@ -675,6 +678,9 @@ per-thread averages: // There's also a combined flag: state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); + + // This says that we process with the rate of state.range(0) bytes every iteration: + state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); ``` When you're compiling in C++11 mode or later you can use `insert()` with diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index c21ccf741e..e6ada560f5 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -373,11 +373,20 @@ class Counter { kAvgIterationsRate = kIsRate | kAvgIterations }; + enum OneK { + // 1'000 items per 1k + kIs1000 = 1000, + // 1'024 items per 1k + kIs1024 = 1024 + }; + double value; Flags flags; + OneK oneK; BENCHMARK_ALWAYS_INLINE - Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {} + Counter(double v = 0., Flags f = kDefaults, OneK k = kIs1000) + : value(v), flags(f), oneK(k) {} BENCHMARK_ALWAYS_INLINE operator double const&() const { return value; } BENCHMARK_ALWAYS_INLINE operator double&() { return value; } diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 48920ca782..e919e78c45 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -150,7 +150,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { for (auto& c : result.counters) { const std::size_t cNameLen = std::max(std::string::size_type(10), c.first.length()); - auto const& s = HumanReadableNumber(c.second.value, 1000); + auto const& s = HumanReadableNumber(c.second.value, c.second.oneK); if (output_options_ & OO_Tabular) { if (c.second.flags & Counter::kIsRate) { printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str()); diff --git a/src/statistics.cc b/src/statistics.cc index 2795011afe..588fd124de 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -165,7 +165,8 @@ std::vector ComputeStats( // user counters for (auto const& kv : counter_stats) { const auto uc_stat = Stat.compute_(kv.second.s); - auto c = Counter(uc_stat, counter_stats[kv.first].c.flags); + auto c = Counter(uc_stat, counter_stats[kv.first].c.flags, + counter_stats[kv.first].c.oneK); data.counters[kv.first] = c; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f49ca5148f..aa2058adce 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -128,6 +128,9 @@ add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) compile_output_test(user_counters_tabular_test) add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +compile_output_test(user_counters_thousands_test) +add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01) + check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) compile_benchmark_test(cxx03_test) diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc new file mode 100644 index 0000000000..6e028291b9 --- /dev/null +++ b/test/user_counters_thousands_test.cc @@ -0,0 +1,153 @@ + +#undef NDEBUG + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// ========================================================================= // +// ------------------------ Thousands Customisation ------------------------ // +// ========================================================================= // + +void BM_Counters_Thousands(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters.insert({ + {"t0_1000000DefaultBase", + bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, + {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1000)}, + {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024)}, + {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1000)}, + {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024)}, + }); +} +BENCHMARK(BM_Counters_Thousands)->Repetitions(2); +ADD_CASES( + TC_ConsoleOut, + { + {"^BM_Counters_Thousands/repeats:2 %console_report " + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2 %console_report " + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_mean %console_report " + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_median %console_report " + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_stddev %console_report " + "t0_1000000DefaultBase=0 t1_1000000Base1000=0 t2_1000000Base1024=0 " + "t3_1048576Base1000=0 t4_1048576Base1024=0$"}, + }); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t4_1048576Base1024\": 0\\.(0)*e\\+(0)*$", MR_Next}, + {"}", MR_Next}}); + +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_Thousands/" + "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+(" + "0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+(" + "0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2_mean\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\." + "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2_median\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\." + "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/repeats:2_stddev\",%csv_report,0,0,0,0,0$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckThousands(Results const& e) { + if (e.name != "BM_Counters_Thousands/repeats:2") + return; // Do not check the aggregates! + + // check that the values are within 0.01% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "t0_1000000DefaultBase", EQ, 1000 * 1000, + 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t1_1000000Base1000", EQ, 1000 * 1000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t2_1000000Base1024", EQ, 1000 * 1000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t3_1048576Base1000", EQ, 1024 * 1024, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t4_1048576Base1024", EQ, 1024 * 1024, 0.0001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Thousands", &CheckThousands); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } From 5159967520f45d2bb31b226b810f4f8dc03df1c4 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 30 Aug 2018 11:59:50 +0300 Subject: [PATCH 077/330] Mark Set{Items,Bytes}Processed()/{items,bytes}_processed() as deprecated. (#654) They are basically proto-version of custom user counters. It does not seem that they do anything that custom user counters don't do. And having two similar entities is not good for generalization. Migration plan: * ``` SetItemsProcessed() => state.counters.insert({ {"", benchmark::Counter(, benchmark::Counter::kIsRate)}, ... }); ``` * ``` SetBytesProcessed() => state.counters.insert({ {"", benchmark::Counter(, benchmark::Counter::kIsRate, benchmark::Counter::OneK::kIs1024)}, ... }); ``` * ``` _processed() => state.counters[""] ``` One thing the custom user counters miss is better support for units of measurement. Refs. https://github.com/google/benchmark/issues/627 --- include/benchmark/benchmark.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index e6ada560f5..adaeeeecd3 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -542,9 +542,17 @@ class State { // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE + BENCHMARK_DEPRECATED_MSG( + "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" + "bytes_processed() will be removed in a future release. " + "Please use custom user counters.") void SetBytesProcessed(int64_t bytes) { bytes_processed_ = bytes; } BENCHMARK_ALWAYS_INLINE + BENCHMARK_DEPRECATED_MSG( + "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" + "bytes_processed() will be removed in a future release. " + "Please use custom user counters.") int64_t bytes_processed() const { return bytes_processed_; } // If this routine is called with complexity_n > 0 and complexity report is @@ -565,9 +573,17 @@ class State { // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE + BENCHMARK_DEPRECATED_MSG( + "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" + "bytes_processed() will be removed in a future release. " + "Please use custom user counters.") void SetItemsProcessed(int64_t items) { items_processed_ = items; } BENCHMARK_ALWAYS_INLINE + BENCHMARK_DEPRECATED_MSG( + "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" + "bytes_processed() will be removed in a future release. " + "Please use custom user counters.") int64_t items_processed() const { return items_processed_; } // If this routine is called, the specified label is printed at the From fbfc495d7f810dbab8d75449acd4fa1ced17eb0b Mon Sep 17 00:00:00 2001 From: pseyfert Date: Mon, 3 Sep 2018 18:45:09 +0200 Subject: [PATCH 078/330] add missing closing bracket in --help message (#666) --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 81a520cdf2..c3f3a6d144 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -595,7 +595,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" - " [--benchmark_report_aggregates_only={true|false}\n" + " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_format=]\n" " [--benchmark_out=]\n" " [--benchmark_out_format=]\n" From 305ba313be0973e83efd70ccc365d284256f6c50 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 4 Sep 2018 13:46:40 -0700 Subject: [PATCH 079/330] Pass name by const-reference instead of by value in class Statistics' constructor (#668) --- include/benchmark/benchmark.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index adaeeeecd3..591b45c36b 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -425,7 +425,7 @@ struct Statistics { std::string name_; StatisticsFunc* compute_; - Statistics(std::string name, StatisticsFunc* compute) + Statistics(const std::string& name, StatisticsFunc* compute) : name_(name), compute_(compute) {} }; From a7ed76ad78c8ad4d0120c236a60b72c7ee39068c Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 5 Sep 2018 14:19:54 +0300 Subject: [PATCH 080/330] Travis-ci: attempt to add 32-bit osx xcode build (#669) Maybe will show https://github.com/google/benchmark/pull/667 or maybe this is completely wrong. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 168ed0b709..4625dfb087 100644 --- a/.travis.yml +++ b/.travis.yml @@ -127,6 +127,11 @@ matrix: compiler: clang env: - COMPILER=clang++ BUILD_TYPE=Release + - os: osx + osx_image: xcode8.3 + compiler: clang + env: + - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON - os: osx osx_image: xcode8.3 compiler: gcc From f0901417c89d123474e6b91365029cfe32cf89dc Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 5 Sep 2018 14:20:18 +0300 Subject: [PATCH 081/330] GetCacheSizesMacOSX(): use consistent types. (#667) I have absolutely no way to test this, but this looks obviously-good. This was reported by Tim Northover @TNorthover in http://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20180903/584223.html > I think this breaks some 32-bit configurations (well, mine at least). > I was using Clang (from Xcode 10 beta) on macOS and got a bunch of > errors referencing sysinfo.cc:292 and onwards: > /Users/tim/llvm/llvm-project/llvm/utils/benchmark/src/sysinfo.cc:292:47: > error: non-constant-expression cannot be narrowed from type > 'std::__1::array::value_type' (aka 'unsigned > long long') to 'size_t' (aka 'unsigned long') in initializer list > [-Wc++11-narrowing] > } Cases[] = {{"hw.l1dcachesize", "Data", 1, CacheCounts[1]}, > ^~~~~~~~~~~~~~ > > The same happens when self-hosting ToT. Unfortunately I couldn't > reproduce the issue on Debian (Clang 6.0.1) even with libc++; I'm not > sure what the difference is. --- src/sysinfo.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 73064b97ba..f79412b542 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -288,7 +288,7 @@ std::vector GetCacheSizesMacOSX() { std::string name; std::string type; int level; - size_t num_sharing; + uint64_t num_sharing; } Cases[] = {{"hw.l1dcachesize", "Data", 1, CacheCounts[1]}, {"hw.l1icachesize", "Instruction", 1, CacheCounts[1]}, {"hw.l2cachesize", "Unified", 2, CacheCounts[2]}, From f274c503e1630739e9619aa72ca08735d01ffc1c Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 10 Sep 2018 23:30:40 +0300 Subject: [PATCH 082/330] Backport LLVM's r341717 "Fix flags used to compile benchmark library with clang-cl" (#673) `MSVC` is true for clang-cl, but `"${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC"` is false, so we would enable -Wall, which means -Weverything with clang-cl, and we get tons of undesired warnings. Use the simpler condition to fix things. Patch by: Reid Kleckner @rnk --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8631199211..fff85b14b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ if (BENCHMARK_BUILD_32_BITS) add_required_cxx_compiler_flag(-m32) endif() -if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") +if (MSVC) # Turn compiler warnings up to 11 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") From c614dfc0d4eadcd19b188ff9c7e226c138f894a1 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 12 Sep 2018 16:26:17 +0300 Subject: [PATCH 083/330] *Display* aggregates only. (#665) There is a flag https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L75-L78 and a call https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/include/benchmark/benchmark.h#L837-L840 But that affects everything, every reporter, destination: https://github.com/google/benchmark/blob/d9cab612e40017af10bddaa5b60c7067032a9e1c/src/benchmark.cc#L316 It would be quite useful to have an ability to be more picky. More specifically, i would like to be able to only see the aggregates in the on-screen output, but for the file output to still contain everything. The former is useful in case of a lot of repetition (or even more so if every iteration is reported separately), while the former is **great** for tooling. Fixes https://github.com/google/benchmark/issues/664 --- README.md | 20 +++++-- include/benchmark/benchmark.h | 22 ++++++- src/benchmark.cc | 90 +++++++++++++++++++++------- src/benchmark_register.cc | 16 +++++ test/CMakeLists.txt | 6 ++ test/display_aggregates_only_test.cc | 40 +++++++++++++ test/output_test.h | 7 +++ test/output_test_helper.cc | 37 ++++++++++++ test/report_aggregates_only_test.cc | 36 +++++++++++ test/reporter_output_test.cc | 27 +++++++++ 10 files changed, 270 insertions(+), 31 deletions(-) create mode 100644 test/display_aggregates_only_test.cc create mode 100644 test/report_aggregates_only_test.cc diff --git a/README.md b/README.md index 2f2d78f6ef..38203958f1 100644 --- a/README.md +++ b/README.md @@ -545,12 +545,20 @@ The number of runs of each benchmark is specified globally by the `Repetitions` on the registered benchmark object. When a benchmark is run more than once the mean, median and standard deviation of the runs will be reported. -Additionally the `--benchmark_report_aggregates_only={true|false}` flag or -`ReportAggregatesOnly(bool)` function can be used to change how repeated tests -are reported. By default the result of each repeated run is reported. When this -option is `true` only the mean, median and standard deviation of the runs is reported. -Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides -the value of the flag for that benchmark. +Additionally the `--benchmark_report_aggregates_only={true|false}`, +`--benchmark_display_aggregates_only={true|false}` flags or +`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be +used to change how repeated tests are reported. By default the result of each +repeated run is reported. When `report aggregates only` option is `true`, +only the aggregates (i.e. mean, median and standard deviation, maybe complexity +measurements if they were requested) of the runs is reported, to both the +reporters - standard output (console), and the file. +However when only the `display aggregates only` option is `true`, +only the aggregates are displayed in the standard output, while the file +output still contains everything. +Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a +registered benchmark object overrides the value of the appropriate flag for that +benchmark. ## User-defined statistics for repeated benchmarks While having mean, median and standard deviation is nice, this may not be diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 591b45c36b..b7a89988cb 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -438,9 +438,21 @@ enum AggregationReportMode : unsigned #else #endif -{ ARM_Unspecified, // The mode has not been manually specified - ARM_Default, // The mode is user-specified as default. - ARM_ReportAggregatesOnly }; +{ + // The mode has not been manually specified + ARM_Unspecified = 0, + // The mode is user-specified. + // This may or may not be set when the following bit-flags are set. + ARM_Default = 1U << 0U, + // File reporter should only output aggregates. + ARM_FileReportAggregatesOnly = 1U << 1U, + // Display reporter should only output aggregates + ARM_DisplayReportAggregatesOnly = 1U << 2U, + // Both reporters should only display aggregates. + ARM_ReportAggregatesOnly = + ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly +}; + } // namespace internal // State is passed to a running Benchmark and contains state for the @@ -862,8 +874,12 @@ class Benchmark { // Specify if each repetition of the benchmark should be reported separately // or if only the final statistics should be reported. If the benchmark // is not repeated then the single result is always reported. + // Applies to *ALL* reporters (display and file). Benchmark* ReportAggregatesOnly(bool value = true); + // Same as ReportAggregatesOnly(), but applies to display reporter only. + Benchmark* DisplayAggregatesOnly(bool value = true); + // If a particular benchmark is I/O bound, runs multiple threads internally or // if for some reason CPU timings are not representative, call this method. If // called, the elapsed time will be used to control how many iterations are diff --git a/src/benchmark.cc b/src/benchmark.cc index c3f3a6d144..0c92ae6aa9 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include "check.h" #include "colorprint.h" @@ -72,10 +73,19 @@ DEFINE_int32(benchmark_repetitions, 1, "The number of runs of each benchmark. If greater than 1, the " "mean and standard deviation of the runs will be reported."); -DEFINE_bool(benchmark_report_aggregates_only, false, - "Report the result of each benchmark repetitions. When 'true' is " - "specified only the mean, standard deviation, and other statistics " - "are reported for repeated benchmarks."); +DEFINE_bool( + benchmark_report_aggregates_only, false, + "Report the result of each benchmark repetitions. When 'true' is specified " + "only the mean, standard deviation, and other statistics are reported for " + "repeated benchmarks. Affects all reporters."); + +DEFINE_bool( + benchmark_display_aggregates_only, false, + "Display the result of each benchmark repetitions. When 'true' is " + "specified only the mean, standard deviation, and other statistics are " + "displayed for repeated benchmarks. Unlike " + "benchmark_report_aggregates_only, only affects the display reporter, but " + "*NOT* file reporter, which will still contain all the output."); DEFINE_string(benchmark_format, "console", "The format to use for console output. Valid values are " @@ -193,10 +203,18 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, manager->NotifyThreadComplete(); } -std::vector RunBenchmark( +struct RunResults { + std::vector non_aggregates; + std::vector aggregates_only; + + bool display_report_aggregates_only = false; + bool file_report_aggregates_only = false; +}; + +RunResults RunBenchmark( const benchmark::internal::Benchmark::Instance& b, std::vector* complexity_reports) { - std::vector reports; // return value + RunResults run_results; const bool has_explicit_iteration_count = b.iterations != 0; size_t iters = has_explicit_iteration_count ? b.iterations : 1; @@ -204,11 +222,20 @@ std::vector RunBenchmark( std::vector pool(b.threads - 1); const int repeats = b.repetitions != 0 ? b.repetitions : FLAGS_benchmark_repetitions; - const bool report_aggregates_only = - repeats != 1 && - (b.aggregation_report_mode == internal::ARM_Unspecified - ? FLAGS_benchmark_report_aggregates_only - : b.aggregation_report_mode == internal::ARM_ReportAggregatesOnly); + if (repeats != 1) { + run_results.display_report_aggregates_only = + (FLAGS_benchmark_report_aggregates_only || + FLAGS_benchmark_display_aggregates_only); + run_results.file_report_aggregates_only = + FLAGS_benchmark_report_aggregates_only; + if (b.aggregation_report_mode != internal::ARM_Unspecified) { + run_results.display_report_aggregates_only = + (b.aggregation_report_mode & + internal::ARM_DisplayReportAggregatesOnly); + run_results.file_report_aggregates_only = + (b.aggregation_report_mode & internal::ARM_FileReportAggregatesOnly); + } + } for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { for (;;) { // Try benchmark @@ -281,7 +308,7 @@ std::vector RunBenchmark( b, results, memory_iterations, memory_result, seconds); if (!report.error_occurred && b.complexity != oNone) complexity_reports->push_back(report); - reports.push_back(report); + run_results.non_aggregates.push_back(report); break; } @@ -304,18 +331,20 @@ std::vector RunBenchmark( iters = static_cast(next_iters + 0.5); } } + // Calculate additional statistics - auto stat_reports = ComputeStats(reports); + run_results.aggregates_only = ComputeStats(run_results.non_aggregates); + + // Maybe calculate complexity report if ((b.complexity != oNone) && b.last_benchmark_instance) { auto additional_run_stats = ComputeBigO(*complexity_reports); - stat_reports.insert(stat_reports.end(), additional_run_stats.begin(), - additional_run_stats.end()); + run_results.aggregates_only.insert(run_results.aggregates_only.end(), + additional_run_stats.begin(), + additional_run_stats.end()); complexity_reports->clear(); } - if (report_aggregates_only) reports.clear(); - reports.insert(reports.end(), stat_reports.begin(), stat_reports.end()); - return reports; + return run_results; } } // namespace @@ -462,11 +491,25 @@ void RunBenchmarks(const std::vector& benchmarks, (!file_reporter || file_reporter->ReportContext(context))) { flushStreams(display_reporter); flushStreams(file_reporter); + for (const auto& benchmark : benchmarks) { - std::vector reports = - RunBenchmark(benchmark, &complexity_reports); - display_reporter->ReportRuns(reports); - if (file_reporter) file_reporter->ReportRuns(reports); + RunResults run_results = RunBenchmark(benchmark, &complexity_reports); + + auto report = [&run_results](BenchmarkReporter* reporter, + bool report_aggregates_only) { + assert(reporter); + assert( + !(report_aggregates_only && run_results.aggregates_only.empty())); + if (!report_aggregates_only) + reporter->ReportRuns(run_results.non_aggregates); + if (!run_results.aggregates_only.empty()) + reporter->ReportRuns(run_results.aggregates_only); + }; + + report(display_reporter, run_results.display_report_aggregates_only); + if (file_reporter) + report(file_reporter, run_results.file_report_aggregates_only); + flushStreams(display_reporter); flushStreams(file_reporter); } @@ -596,6 +639,7 @@ void PrintUsageAndExit() { " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" " [--benchmark_report_aggregates_only={true|false}]\n" + " [--benchmark_display_aggregates_only={true|false}]\n" " [--benchmark_format=]\n" " [--benchmark_out=]\n" " [--benchmark_out_format=]\n" @@ -619,6 +663,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_repetitions) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", &FLAGS_benchmark_report_aggregates_only) || + ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", + &FLAGS_benchmark_display_aggregates_only) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || ParseStringFlag(argv[i], "benchmark_out_format", diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 8ee30adf0f..ffe97a1416 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -373,6 +373,22 @@ Benchmark* Benchmark::ReportAggregatesOnly(bool value) { return this; } +Benchmark* Benchmark::DisplayAggregatesOnly(bool value) { + // If we were called, the report mode is no longer 'unspecified', in any case. + aggregation_report_mode_ = static_cast( + aggregation_report_mode_ | ARM_Default); + + if (value) { + aggregation_report_mode_ = static_cast( + aggregation_report_mode_ | ARM_DisplayReportAggregatesOnly); + } else { + aggregation_report_mode_ = static_cast( + aggregation_report_mode_ & ~ARM_DisplayReportAggregatesOnly); + } + + return this; +} + Benchmark* Benchmark::UseRealTime() { CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index aa2058adce..4269991755 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -125,6 +125,12 @@ add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01 compile_output_test(user_counters_test) add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) +compile_output_test(report_aggregates_only_test) +add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01) + +compile_output_test(display_aggregates_only_test) +add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01) + compile_output_test(user_counters_tabular_test) add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) diff --git a/test/display_aggregates_only_test.cc b/test/display_aggregates_only_test.cc new file mode 100644 index 0000000000..46ab2e310f --- /dev/null +++ b/test/display_aggregates_only_test.cc @@ -0,0 +1,40 @@ + +#undef NDEBUG +#include +#include + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// Ok this test is super ugly. We want to check what happens with the file +// reporter in the presence of DisplayAggregatesOnly(). +// We do not care about console output, the normal tests check that already. + +void BM_SummaryRepeat(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly(); + +int main(int argc, char* argv[]) { + const std::string output = GetFileReporterOutput(argc, argv); + + if (SubstrCnt(output, "BM_SummaryRepeat/repeats:3") != 6 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3\"") != 3 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_median\"") != 1 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_stddev\"") != 1) { + std::cout << "Precondition mismatch. Expected to only find 6 " + "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" + "\"BM_SummaryRepeat/repeats:3\", " + "\"BM_SummaryRepeat/repeats:3\", " + "\"BM_SummaryRepeat/repeats:3\", " + "\"BM_SummaryRepeat/repeats:3_mean\", " + "\"BM_SummaryRepeat/repeats:3_median\", " + "\"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire output:\n"; + std::cout << output; + return 1; + } + + return 0; +} diff --git a/test/output_test.h b/test/output_test.h index 31a919991f..9385761b21 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -60,6 +60,13 @@ int SetSubstitutions( // Run all output tests. void RunOutputTests(int argc, char* argv[]); +// Count the number of 'pat' substrings in the 'haystack' string. +int SubstrCnt(const std::string& haystack, const std::string& pat); + +// Run registered benchmarks with file reporter enabled, and return the content +// outputted by the file reporter. +std::string GetFileReporterOutput(int argc, char* argv[]); + // ========================================================================= // // ------------------------- Results checking ------------------------------ // // ========================================================================= // diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 394c4f5d1a..f84bd34521 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -1,8 +1,11 @@ +#include #include +#include #include #include #include #include +#include #include "../src/benchmark_api_internal.h" #include "../src/check.h" // NOTE: check.h is for internal use only! @@ -423,3 +426,37 @@ void RunOutputTests(int argc, char* argv[]) { CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); } + +int SubstrCnt(const std::string& haystack, const std::string& pat) { + if (pat.length() == 0) return 0; + int count = 0; + for (size_t offset = haystack.find(pat); offset != std::string::npos; + offset = haystack.find(pat, offset + pat.length())) + ++count; + return count; +} + +std::string GetFileReporterOutput(int argc, char* argv[]) { + std::vector new_argv(argv, argv + argc); + assert(static_cast(argc) == new_argv.size()); + + std::string tmp_file_name = std::tmpnam(nullptr); + std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; + + std::string tmp = "--benchmark_out="; + tmp += tmp_file_name; + new_argv.emplace_back(const_cast(tmp.c_str())); + + argc = int(new_argv.size()); + + benchmark::Initialize(&argc, new_argv.data()); + benchmark::RunSpecifiedBenchmarks(); + + // Read the output back from the file, and delete the file. + std::ifstream tmp_stream(tmp_file_name); + std::string output = std::string((std::istreambuf_iterator(tmp_stream)), + std::istreambuf_iterator()); + std::remove(tmp_file_name.c_str()); + + return output; +} diff --git a/test/report_aggregates_only_test.cc b/test/report_aggregates_only_test.cc new file mode 100644 index 0000000000..8eb8bde2e2 --- /dev/null +++ b/test/report_aggregates_only_test.cc @@ -0,0 +1,36 @@ + +#undef NDEBUG +#include +#include + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// Ok this test is super ugly. We want to check what happens with the file +// reporter in the presence of ReportAggregatesOnly(). +// We do not care about console output, the normal tests check that already. + +void BM_SummaryRepeat(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); + +int main(int argc, char* argv[]) { + const std::string output = GetFileReporterOutput(argc, argv); + + if (SubstrCnt(output, "BM_SummaryRepeat/repeats:3") != 3 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_median\"") != 1 || + SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_stddev\"") != 1) { + std::cout << "Precondition mismatch. Expected to only find three " + "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" + "\"BM_SummaryRepeat/repeats:3_mean\", " + "\"BM_SummaryRepeat/repeats:3_median\", " + "\"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire output:\n"; + std::cout << output; + return 1; + } + + return 0; +} diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index a98fc42a45..8045cfc9ea 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -342,6 +342,33 @@ ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); +// Test that non-aggregate data is not displayed. +// NOTE: this test is kinda bad. we are only testing the display output. +// But we don't check that the file output still contains everything... +void BM_SummaryDisplay(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); +ADD_CASES(TC_ConsoleOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"^BM_SummaryDisplay/repeats:2_mean %console_report$"}, + {"^BM_SummaryDisplay/repeats:2_median %console_report$"}, + {"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}}); +ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, + {"\"run_type\": \"aggregate\",$", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, + {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"}, + {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}}); + +// Test repeats with custom time unit. void BM_RepeatTimeUnit(benchmark::State& state) { for (auto _ : state) { } From 58588476ce52c0af4cb54f9c596c4579f78bc953 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 13 Sep 2018 15:08:15 +0300 Subject: [PATCH 084/330] Track two more details about runs - the aggregate name, and run name. (#675) This is related to @BaaMeow's work in https://github.com/google/benchmark/pull/616 but is not based on it. Two new fields are tracked, and dumped into JSON: * If the run is an aggregate, the aggregate's name is stored. It can be RMS, BigO, mean, median, stddev, or any custom stat name. * The aggregate-name-less run name is additionally stored. I.e. not some name of the benchmark function, but the actual name, but without the 'aggregate name' suffix. This way one can group/filter all the runs, and filter by the particular aggregate type. I *might* need this for further tooling improvement. Or maybe not. But this is certainly worthwhile for custom tooling. --- include/benchmark/benchmark.h | 4 +- src/benchmark.cc | 2 +- src/complexity.cc | 11 ++- src/console_reporter.cc | 2 +- src/csv_reporter.cc | 2 +- src/json_reporter.cc | 6 +- src/reporter.cc | 8 ++ src/statistics.cc | 5 +- test/CMakeLists.txt | 3 + test/complexity_test.cc | 37 +++++--- test/display_aggregates_only_test.cc | 25 +++--- test/memory_manager_test.cc | 1 + test/register_benchmark_test.cc | 4 +- test/report_aggregates_only_test.cc | 17 ++-- test/reporter_output_test.cc | 129 +++++++++++++++++++------- test/skip_with_error_test.cc | 4 +- test/user_counters_tabular_test.cc | 123 +++++++++++++------------ test/user_counters_test.cc | 130 +++++++++++++++------------ test/user_counters_thousands_test.cc | 8 ++ 19 files changed, 334 insertions(+), 187 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index b7a89988cb..7aa6487006 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1339,8 +1339,10 @@ class BenchmarkReporter { allocs_per_iter(0.0), max_bytes_used(0) {} - std::string benchmark_name; + std::string benchmark_name() const; + std::string run_name; RunType run_type; // is this a measurement, or an aggregate? + std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. bool error_occurred; std::string error_message; diff --git a/src/benchmark.cc b/src/benchmark.cc index 0c92ae6aa9..4d3001a044 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -132,7 +132,7 @@ BenchmarkReporter::Run CreateRunReport( // Create report about this benchmark run. BenchmarkReporter::Run report; - report.benchmark_name = b.name; + report.run_name = b.name; report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; diff --git a/src/complexity.cc b/src/complexity.cc index 8d1392a3cc..0be73e0822 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -182,13 +182,15 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); } - std::string benchmark_name = - reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + + std::string run_name = reports[0].benchmark_name().substr( + 0, reports[0].benchmark_name().find('/')); // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; + big_o.run_name = run_name; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; - big_o.benchmark_name = benchmark_name + "_BigO"; + big_o.aggregate_name = "BigO"; big_o.iterations = 0; big_o.real_accumulated_time = result_real.coef; big_o.cpu_accumulated_time = result_cpu.coef; @@ -204,9 +206,10 @@ std::vector ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; + rms.run_name = run_name; big_o.report_label = reports[0].report_label; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; - rms.benchmark_name = benchmark_name + "_RMS"; + rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; rms.iterations = 0; rms.real_accumulated_time = result_real.rms / multiplier; diff --git a/src/console_reporter.cc b/src/console_reporter.cc index e919e78c45..f1d872a435 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -106,7 +106,7 @@ void ConsoleReporter::PrintRunData(const Run& result) { auto name_color = (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; printer(Out, name_color, "%-*s ", name_field_width_, - result.benchmark_name.c_str()); + result.benchmark_name().c_str()); if (result.error_occurred) { printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index 4a641909d8..dc370defa0 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -88,7 +88,7 @@ void CSVReporter::PrintRunData(const Run& run) { // Field with embedded double-quote characters must be doubled and the field // delimited with double-quotes. - std::string name = run.benchmark_name; + std::string name = run.benchmark_name(); ReplaceAll(&name, "\"", "\"\""); Out << '"' << name << "\","; if (run.error_occurred) { diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 54e3af38d9..475cf0a1a8 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -159,7 +159,8 @@ void JSONReporter::Finalize() { void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); - out << indent << FormatKV("name", run.benchmark_name) << ",\n"; + out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; + out << indent << FormatKV("run_name", run.run_name) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { case BenchmarkReporter::Run::RT_Iteration: @@ -169,6 +170,9 @@ void JSONReporter::PrintRunData(Run const& run) { } BENCHMARK_UNREACHABLE(); }()) << ",\n"; + if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) { + out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n"; + } if (run.error_occurred) { out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; out << indent << FormatKV("error_message", run.error_message) << ",\n"; diff --git a/src/reporter.cc b/src/reporter.cc index 541661a25f..fe86617717 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -72,6 +72,14 @@ const char *BenchmarkReporter::Context::executable_name; BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()) {} +std::string BenchmarkReporter::Run::benchmark_name() const { + std::string name = run_name; + if (run_type == RT_Aggregate) { + name += "_" + aggregate_name; + } + return name; +} + double BenchmarkReporter::Run::GetAdjustedRealTime() const { double new_time = real_accumulated_time * GetTimeUnitMultiplier(time_unit); if (iterations != 0) new_time /= static_cast(iterations); diff --git a/src/statistics.cc b/src/statistics.cc index 588fd124de..b5fec520b6 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -123,7 +123,7 @@ std::vector ComputeStats( // Populate the accumulators. for (Run const& run : reports) { - CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); + CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); CHECK_EQ(run_iterations, run.iterations); if (run.error_occurred) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); @@ -150,8 +150,9 @@ std::vector ComputeStats( for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; + data.run_name = reports[0].benchmark_name(); data.run_type = BenchmarkReporter::Run::RT_Aggregate; - data.benchmark_name = reports[0].benchmark_name + "_" + Stat.name_; + data.aggregate_name = Stat.name_; data.report_label = report_label; data.iterations = run_iterations; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4269991755..f15ce20818 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -137,6 +137,9 @@ add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_count compile_output_test(user_counters_thousands_test) add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01) +compile_output_test(memory_manager_test) +add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01) + check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) compile_benchmark_test(cxx03_test) diff --git a/test/complexity_test.cc b/test/complexity_test.cc index d09a2f8d65..323ddfe7ac 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -12,9 +12,10 @@ namespace { #define ADD_COMPLEXITY_CASES(...) \ int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) -int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, - std::string big_o) { - SetSubstitutions({{"%bigo_name", big_o_test_name}, +int AddComplexityTest(std::string test_name, std::string big_o_test_name, + std::string rms_test_name, std::string big_o) { + SetSubstitutions({{"%name", test_name}, + {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, @@ -25,14 +26,18 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, + {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, + {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"RMS\",$", MR_Next}, {"\"rms\": %float$", MR_Next}, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, @@ -61,6 +66,7 @@ BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) { return 1.0; }); +const char *one_test_name = "BM_Complexity_O1"; const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; const char *enum_big_o_1 = "\\([0-9]+\\)"; @@ -71,13 +77,16 @@ const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + enum_big_o_1); // Add auto enum tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + auto_big_o_1); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + lambda_big_o_1); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -114,16 +123,19 @@ BENCHMARK(BM_Complexity_O_N) ->Range(1 << 10, 1 << 16) ->Complexity(); +const char *n_test_name = "BM_Complexity_O_N"; const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; const char *enum_auto_big_o_n = "N"; const char *lambda_big_o_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + enum_auto_big_o_n); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + lambda_big_o_n); // ========================================================================= // // ------------------------- Testing BigO O(N*lgN) ------------------------- // @@ -150,18 +162,19 @@ BENCHMARK(BM_Complexity_O_N_log_N) ->Range(1 << 10, 1 << 16) ->Complexity(); +const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; const char *enum_auto_big_o_n_lg_n = "NlgN"; const char *lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, - enum_auto_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, - lambda_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/test/display_aggregates_only_test.cc b/test/display_aggregates_only_test.cc index 46ab2e310f..3c36d3f03c 100644 --- a/test/display_aggregates_only_test.cc +++ b/test/display_aggregates_only_test.cc @@ -19,19 +19,22 @@ BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly(); int main(int argc, char* argv[]) { const std::string output = GetFileReporterOutput(argc, argv); - if (SubstrCnt(output, "BM_SummaryRepeat/repeats:3") != 6 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3\"") != 3 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_mean\"") != 1 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_median\"") != 1 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_stddev\"") != 1) { + if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 6 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3\"") != 3 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != + 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != + 1) { std::cout << "Precondition mismatch. Expected to only find 6 " "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" - "\"BM_SummaryRepeat/repeats:3\", " - "\"BM_SummaryRepeat/repeats:3\", " - "\"BM_SummaryRepeat/repeats:3\", " - "\"BM_SummaryRepeat/repeats:3_mean\", " - "\"BM_SummaryRepeat/repeats:3_median\", " - "\"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire output:\n"; + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_median\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " + "output:\n"; std::cout << output; return 1; } diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 4e71b649a4..94be608379 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -21,6 +21,7 @@ BENCHMARK(BM_empty); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc index 18de6d68e2..3ac5b21fb3 100644 --- a/test/register_benchmark_test.cc +++ b/test/register_benchmark_test.cc @@ -30,8 +30,8 @@ struct TestCase { void CheckRun(Run const& run) const { // clang-format off - CHECK(name == run.benchmark_name) << "expected " << name << " got " - << run.benchmark_name; + CHECK(name == run.benchmark_name()) << "expected " << name << " got " + << run.benchmark_name(); if (label) { CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; diff --git a/test/report_aggregates_only_test.cc b/test/report_aggregates_only_test.cc index 8eb8bde2e2..9646b9be53 100644 --- a/test/report_aggregates_only_test.cc +++ b/test/report_aggregates_only_test.cc @@ -19,15 +19,18 @@ BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); int main(int argc, char* argv[]) { const std::string output = GetFileReporterOutput(argc, argv); - if (SubstrCnt(output, "BM_SummaryRepeat/repeats:3") != 3 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_mean\"") != 1 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_median\"") != 1 || - SubstrCnt(output, "\"BM_SummaryRepeat/repeats:3_stddev\"") != 1) { + if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 3 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != + 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != + 1) { std::cout << "Precondition mismatch. Expected to only find three " "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" - "\"BM_SummaryRepeat/repeats:3_mean\", " - "\"BM_SummaryRepeat/repeats:3_median\", " - "\"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire output:\n"; + "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_median\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " + "output:\n"; std::cout << output; return 1; } diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8045cfc9ea..91e505d5dc 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -65,6 +65,7 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -87,6 +88,7 @@ BENCHMARK(BM_bytes_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -110,6 +112,7 @@ BENCHMARK(BM_items_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -132,6 +135,7 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -154,6 +158,7 @@ void BM_error(benchmark::State& state) { BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, {"\"error_message\": \"message\",$", MR_Next}}); @@ -172,6 +177,7 @@ void BM_no_arg_name(benchmark::State& state) { BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, + {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); @@ -186,6 +192,7 @@ void BM_arg_name(benchmark::State& state) { BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, + {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); @@ -200,8 +207,10 @@ void BM_arg_names(benchmark::State& state) { BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, + {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); // ========================================================================= // @@ -236,15 +245,23 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"}, {"^BM_Repeat/repeats:2_median %console_report$"}, {"^BM_Repeat/repeats:2_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, @@ -259,17 +276,26 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, {"^BM_Repeat/repeats:3_median %console_report$"}, {"^BM_Repeat/repeats:3_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -286,19 +312,29 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"}, {"^BM_Repeat/repeats:4_median %console_report$"}, {"^BM_Repeat/repeats:4_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -316,6 +352,7 @@ void BM_RepeatOnce(benchmark::State& state) { BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, + {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); @@ -330,13 +367,20 @@ ADD_CASES(TC_ConsoleOut, {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, {"^BM_SummaryRepeat/repeats:3_median %console_report$"}, {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); -ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, @@ -355,13 +399,20 @@ ADD_CASES(TC_ConsoleOut, {"^BM_SummaryDisplay/repeats:2_mean %console_report$"}, {"^BM_SummaryDisplay/repeats:2_median %console_report$"}, {"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}}); -ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, - {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, @@ -382,16 +433,23 @@ ADD_CASES(TC_ConsoleOut, {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"}, {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); -ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"time_unit\": \"us\",?$"}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"time_unit\": \"us\",?$"}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"time_unit\": \"us\",?$"}}); +ADD_CASES(TC_JSONOut, + {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"time_unit\": \"us\",?$"}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"time_unit\": \"us\",?$"}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"time_unit\": \"us\",?$"}}); ADD_CASES(TC_CSVOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, @@ -425,19 +483,30 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, {"^BM_UserStats/repeats:3_stddev %console_report$"}, {"^BM_UserStats/repeats:3_ %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_median\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"name\": \"BM_UserStats/repeats:3_\",$"}, - {"\"run_type\": \"aggregate\",$", MR_Next}}); + {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"}, {"^\"BM_UserStats/repeats:3\",%csv_report$"}, diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 39785fb7f6..06579772ff 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -33,8 +33,8 @@ struct TestCase { typedef benchmark::BenchmarkReporter::Run Run; void CheckRun(Run const& run) const { - CHECK(name == run.benchmark_name) - << "expected " << name << " got " << run.benchmark_name; + CHECK(name == run.benchmark_name()) + << "expected " << name << " got " << run.benchmark_name(); CHECK(error_occurred == run.error_occurred); CHECK(error_message == run.error_message); if (error_occurred) { diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 88cb3d23ef..030e98916c 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -69,19 +69,21 @@ void BM_Counters_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float,$", MR_Next}, - {"\"Frob\": %float,$", MR_Next}, - {"\"Lob\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -114,19 +116,22 @@ void BM_CounterRates_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float,$", MR_Next}, - {"\"Frob\": %float,$", MR_Next}, - {"\"Lob\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -159,16 +164,18 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," "%float,,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -192,16 +199,18 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," "%float,,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -229,16 +238,18 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," ",%float,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 2c7a7b11b7..57d07d9138 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -70,17 +71,19 @@ BENCHMARK(BM_Counters_WithBytesAndItemsPSec); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bytes_per_second\": %float,$", MR_Next}, - {"\"items_per_second\": %float,$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bytes_per_second\": %float,$", MR_Next}, + {"\"items_per_second\": %float,$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," "%csv_bytes_items_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -112,6 +115,7 @@ ADD_CASES( TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -144,15 +148,17 @@ void BM_Counters_Threads(benchmark::State& state) { BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES( TC_CSVOut, {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); @@ -178,15 +184,17 @@ void BM_Counters_AvgThreads(benchmark::State& state) { BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " "%console_report bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES( TC_CSVOut, {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); @@ -215,6 +223,8 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", + MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -248,15 +258,17 @@ void BM_Counters_IterationInvariant(benchmark::State& state) { BENCHMARK(BM_Counters_IterationInvariant); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -288,6 +300,8 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", + MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -324,15 +338,17 @@ void BM_Counters_AvgIterations(benchmark::State& state) { BENCHMARK(BM_Counters_AvgIterations); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -360,15 +376,17 @@ void BM_Counters_kAvgIterationsRate(benchmark::State& state) { BENCHMARK(BM_Counters_kAvgIterationsRate); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report," "%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 6e028291b9..2c64dc4207 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -51,6 +51,7 @@ ADD_CASES( }); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -64,6 +65,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -77,7 +79,9 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -90,7 +94,9 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -103,7 +109,9 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, From 1b44120cd16712f3b5decd95dc8ff2813574b273 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 13 Sep 2018 22:03:47 +0300 Subject: [PATCH 085/330] Un-deprecate [SG]et{Item,Byte}sProcessed, re-implement as custom counters. (#676) As discussed with @dominichamon and @dbabokin, sugar is nice. Well, maybe not for the health, but it's sweet. Alright, enough puns. A special care needs to be applied not to break csv reporter. UGH. We end up shedding some code over this. We no longer specially pretty-print them, they are printed just like the rest of custom counters. Fixes #627. --- include/benchmark/benchmark.h | 50 +++++++++++++---------------------- src/benchmark.cc | 15 ----------- src/console_reporter.cc | 20 -------------- src/csv_reporter.cc | 12 ++++++--- src/json_reporter.cc | 9 +------ src/statistics.cc | 8 ------ src/thread_manager.h | 2 -- test/reporter_output_test.cc | 8 +++--- test/user_counters_test.cc | 12 ++++----- 9 files changed, 37 insertions(+), 99 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 7aa6487006..04fa202fdf 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -548,24 +548,21 @@ class State { // Set the number of bytes processed by the current benchmark // execution. This routine is typically called once at the end of a - // throughput oriented benchmark. If this routine is called with a - // value > 0, the report is printed in MB/sec instead of nanoseconds - // per iteration. + // throughput oriented benchmark. // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE - BENCHMARK_DEPRECATED_MSG( - "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" - "bytes_processed() will be removed in a future release. " - "Please use custom user counters.") - void SetBytesProcessed(int64_t bytes) { bytes_processed_ = bytes; } + void SetBytesProcessed(int64_t bytes) { + counters["bytes_per_second"] = + Counter(bytes, Counter::kIsRate, Counter::kIs1024); + } BENCHMARK_ALWAYS_INLINE - BENCHMARK_DEPRECATED_MSG( - "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" - "bytes_processed() will be removed in a future release. " - "Please use custom user counters.") - int64_t bytes_processed() const { return bytes_processed_; } + int64_t bytes_processed() const { + if (counters.find("bytes_per_second") != counters.end()) + return counters.at("bytes_per_second"); + return 0; + } // If this routine is called with complexity_n > 0 and complexity report is // requested for the @@ -585,18 +582,16 @@ class State { // // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE - BENCHMARK_DEPRECATED_MSG( - "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" - "bytes_processed() will be removed in a future release. " - "Please use custom user counters.") - void SetItemsProcessed(int64_t items) { items_processed_ = items; } + void SetItemsProcessed(int64_t items) { + counters["items_per_second"] = Counter(items, benchmark::Counter::kIsRate); + } BENCHMARK_ALWAYS_INLINE - BENCHMARK_DEPRECATED_MSG( - "The SetItemsProcessed()/items_processed()/SetBytesProcessed()/" - "bytes_processed() will be removed in a future release. " - "Please use custom user counters.") - int64_t items_processed() const { return items_processed_; } + int64_t items_processed() const { + if (counters.find("items_per_second") != counters.end()) + return counters.at("items_per_second"); + return 0; + } // If this routine is called, the specified label is printed at the // end of the benchmark report line for the currently executing @@ -659,9 +654,6 @@ class State { private: // items we don't need on the first cache line std::vector range_; - int64_t bytes_processed_; - int64_t items_processed_; - int64_t complexity_n_; public: @@ -1326,8 +1318,6 @@ class BenchmarkReporter { time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), - bytes_per_second(0), - items_per_second(0), max_heapbytes_used(0), complexity(oNone), complexity_lambda(), @@ -1364,10 +1354,6 @@ class BenchmarkReporter { // accumulated time. double GetAdjustedCPUTime() const; - // Zero if not set by benchmark. - double bytes_per_second; - double items_per_second; - // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; diff --git a/src/benchmark.cc b/src/benchmark.cc index 4d3001a044..dcdd23f40e 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -141,23 +141,12 @@ BenchmarkReporter::Run CreateRunReport( report.time_unit = b.time_unit; if (!report.error_occurred) { - double bytes_per_second = 0; - if (results.bytes_processed > 0 && seconds > 0.0) { - bytes_per_second = (results.bytes_processed / seconds); - } - double items_per_second = 0; - if (results.items_processed > 0 && seconds > 0.0) { - items_per_second = (results.items_processed / seconds); - } - if (b.use_manual_time) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } report.cpu_accumulated_time = results.cpu_time_used; - report.bytes_per_second = bytes_per_second; - report.items_per_second = items_per_second; report.complexity_n = results.complexity_n; report.complexity = b.complexity; report.complexity_lambda = b.complexity_lambda; @@ -195,8 +184,6 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, results.cpu_time_used += timer.cpu_time_used(); results.real_time_used += timer.real_time_used(); results.manual_time_used += timer.manual_time_used(); - results.bytes_processed += st.bytes_processed(); - results.items_processed += st.items_processed(); results.complexity_n += st.complexity_length_n(); internal::Increment(&results.counters, st.counters); } @@ -360,8 +347,6 @@ State::State(size_t max_iters, const std::vector& ranges, int thread_i, finished_(false), error_occurred_(false), range_(ranges), - bytes_processed_(0), - items_processed_(0), complexity_n_(0), counters(), thread_index(thread_i), diff --git a/src/console_reporter.cc b/src/console_reporter.cc index f1d872a435..7de71386fe 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -114,18 +114,6 @@ void ConsoleReporter::PrintRunData(const Run& result) { printer(Out, COLOR_DEFAULT, "\n"); return; } - // Format bytes per second - std::string rate; - if (result.bytes_per_second > 0) { - rate = StrCat(" ", HumanReadableNumber(result.bytes_per_second), "B/s"); - } - - // Format items per second - std::string items; - if (result.items_per_second > 0) { - items = - StrCat(" ", HumanReadableNumber(result.items_per_second), " items/s"); - } const double real_time = result.GetAdjustedRealTime(); const double cpu_time = result.GetAdjustedCPUTime(); @@ -164,14 +152,6 @@ void ConsoleReporter::PrintRunData(const Run& result) { } } - if (!rate.empty()) { - printer(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); - } - - if (!items.empty()) { - printer(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); - } - if (!result.report_label.empty()) { printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); } diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index dc370defa0..d2f1d27eb6 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -49,6 +49,8 @@ void CSVReporter::ReportRuns(const std::vector& reports) { // save the names of all the user counters for (const auto& run : reports) { for (const auto& cnt : run.counters) { + if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + continue; user_counter_names_.insert(cnt.first); } } @@ -69,6 +71,8 @@ void CSVReporter::ReportRuns(const std::vector& reports) { // check that all the current counters are saved in the name set for (const auto& run : reports) { for (const auto& cnt : run.counters) { + if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + continue; CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) << "All counters must be present in each run. " << "Counter named \"" << cnt.first @@ -117,12 +121,12 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - if (run.bytes_per_second > 0.0) { - Out << run.bytes_per_second; + if (run.counters.find("bytes_per_second") != run.counters.end()) { + Out << run.counters.at("bytes_per_second"); } Out << ","; - if (run.items_per_second > 0.0) { - Out << run.items_per_second; + if (run.counters.find("items_per_second") != run.counters.end()) { + Out << run.counters.at("items_per_second"); } Out << ","; if (!run.report_label.empty()) { diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 475cf0a1a8..6866c713cf 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -193,14 +193,7 @@ void JSONReporter::PrintRunData(Run const& run) { } else if (run.report_rms) { out << indent << FormatKV("rms", run.GetAdjustedCPUTime()); } - if (run.bytes_per_second > 0.0) { - out << ",\n" - << indent << FormatKV("bytes_per_second", run.bytes_per_second); - } - if (run.items_per_second > 0.0) { - out << ",\n" - << indent << FormatKV("items_per_second", run.items_per_second); - } + for (auto& c : run.counters) { out << ",\n" << indent << FormatKV(c.first, c.second); } diff --git a/src/statistics.cc b/src/statistics.cc index b5fec520b6..8c90a44f01 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -91,13 +91,9 @@ std::vector ComputeStats( // Accumulators. std::vector real_accumulated_time_stat; std::vector cpu_accumulated_time_stat; - std::vector bytes_per_second_stat; - std::vector items_per_second_stat; real_accumulated_time_stat.reserve(reports.size()); cpu_accumulated_time_stat.reserve(reports.size()); - bytes_per_second_stat.reserve(reports.size()); - items_per_second_stat.reserve(reports.size()); // All repetitions should be run with the same number of iterations so we // can take this information from the first benchmark. @@ -128,8 +124,6 @@ std::vector ComputeStats( if (run.error_occurred) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); - items_per_second_stat.emplace_back(run.items_per_second); - bytes_per_second_stat.emplace_back(run.bytes_per_second); // user counters for (auto const& cnt : run.counters) { auto it = counter_stats.find(cnt.first); @@ -158,8 +152,6 @@ std::vector ComputeStats( data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat); data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat); - data.bytes_per_second = Stat.compute_(bytes_per_second_stat); - data.items_per_second = Stat.compute_(items_per_second_stat); data.time_unit = reports[0].time_unit; diff --git a/src/thread_manager.h b/src/thread_manager.h index 82b4d72b62..6e274c7ea6 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -42,8 +42,6 @@ class ThreadManager { double real_time_used = 0; double cpu_time_used = 0; double manual_time_used = 0; - int64_t bytes_processed = 0; - int64_t items_processed = 0; int64_t complexity_n = 0; std::string report_label_; std::string error_message_; diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 91e505d5dc..50c1f758ae 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -85,8 +85,8 @@ void BM_bytes_per_second(benchmark::State& state) { } BENCHMARK(BM_bytes_per_second); -ADD_CASES(TC_ConsoleOut, - {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " + "bytes_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -109,8 +109,8 @@ void BM_items_per_second(benchmark::State& state) { } BENCHMARK(BM_items_per_second); -ADD_CASES(TC_ConsoleOut, - {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " + "items_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 57d07d9138..bb0d6b4c5a 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -68,9 +68,9 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { state.SetItemsProcessed(150); } BENCHMARK(BM_Counters_WithBytesAndItemsPSec); -ADD_CASES(TC_ConsoleOut, - {{"^BM_Counters_WithBytesAndItemsPSec %console_report " - "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " + "bar=%hrfloat bytes_per_second=%hrfloat/s " + "foo=%hrfloat items_per_second=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, @@ -79,10 +79,10 @@ ADD_CASES(TC_JSONOut, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bytes_per_second\": %float,$", MR_Next}, - {"\"items_per_second\": %float,$", MR_Next}, {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, + {"\"bytes_per_second\": %float,$", MR_Next}, + {"\"foo\": %float,$", MR_Next}, + {"\"items_per_second\": %float$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," "%csv_bytes_items_report,%float,%float$"}}); From a5e9c061d9363403c93d16cf79d16f30f94063f6 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 17 Sep 2018 11:59:39 +0300 Subject: [PATCH 086/330] [Tooling] 'display aggregates only' support. (#674) Works as you'd expect, both for the benchmark binaries, and the jsons (naturally, since the tests need to work): ``` $ ~/src/googlebenchmark/tools/compare.py benchmarks ~/rawspeed/build-{0,1}/src/utilities/rsbench/rsbench --benchmark_repetitions=3 * RUNNING: /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_out=/tmp/tmpYoui5H 2018-09-12 20:23:47 Running /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench Run on (8 X 4000 MHz CPU s) CPU Caches: L1 Data 16K (x8) L1 Instruction 64K (x4) L2 Unified 2048K (x4) L3 Unified 8192K (x1) ----------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... ----------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time 447 ms 447 ms 2 CPUTime,s=0.447302 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=117.738M Pixels/WallTime=117.738M Raws/CPUTime=2.23562 Raws/WallTime=2.23564 WallTime,s=0.4473 2K4A9927.CR2/threads:8/real_time 444 ms 444 ms 2 CPUTime,s=0.444228 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.552M Pixels/WallTime=118.553M Raws/CPUTime=2.2511 Raws/WallTime=2.25111 WallTime,s=0.444226 2K4A9927.CR2/threads:8/real_time 447 ms 447 ms 2 CPUTime,s=0.446983 CPUTime/WallTime=0.999999 Pixels=52.6643M Pixels/CPUTime=117.822M Pixels/WallTime=117.822M Raws/CPUTime=2.23722 Raws/WallTime=2.23722 WallTime,s=0.446984 2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446171 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.037M Pixels/WallTime=118.038M Raws/CPUTime=2.24131 Raws/WallTime=2.24132 WallTime,s=0.44617 2K4A9927.CR2/threads:8/real_time_median 447 ms 447 ms 2 CPUTime,s=0.446983 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=117.822M Pixels/WallTime=117.822M Raws/CPUTime=2.23722 Raws/WallTime=2.23722 WallTime,s=0.446984 2K4A9927.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=1.69052m CPUTime/WallTime=3.53737u Pixels=0 Pixels/CPUTime=448.178k Pixels/WallTime=448.336k Raws/CPUTime=8.51008m Raws/WallTime=8.51308m WallTime,s=1.6911m 2K4A9928.CR2/threads:8/real_time 563 ms 563 ms 1 CPUTime,s=0.562511 CPUTime/WallTime=0.999824 Pixels=27.9936M Pixels/CPUTime=49.7654M Pixels/WallTime=49.7567M Raws/CPUTime=1.77774 Raws/WallTime=1.77743 WallTime,s=0.56261 2K4A9928.CR2/threads:8/real_time 561 ms 561 ms 1 CPUTime,s=0.561328 CPUTime/WallTime=0.999917 Pixels=27.9936M Pixels/CPUTime=49.8703M Pixels/WallTime=49.8662M Raws/CPUTime=1.78149 Raws/WallTime=1.78134 WallTime,s=0.561375 2K4A9928.CR2/threads:8/real_time 570 ms 570 ms 1 CPUTime,s=0.570423 CPUTime/WallTime=0.999876 Pixels=27.9936M Pixels/CPUTime=49.0752M Pixels/WallTime=49.0691M Raws/CPUTime=1.75308 Raws/WallTime=1.75287 WallTime,s=0.570493 2K4A9928.CR2/threads:8/real_time_mean 565 ms 565 ms 1 CPUTime,s=0.564754 CPUTime/WallTime=0.999872 Pixels=27.9936M Pixels/CPUTime=49.5703M Pixels/WallTime=49.564M Raws/CPUTime=1.77077 Raws/WallTime=1.77055 WallTime,s=0.564826 2K4A9928.CR2/threads:8/real_time_median 563 ms 563 ms 1 CPUTime,s=0.562511 CPUTime/WallTime=0.999876 Pixels=27.9936M Pixels/CPUTime=49.7654M Pixels/WallTime=49.7567M Raws/CPUTime=1.77774 Raws/WallTime=1.77743 WallTime,s=0.56261 2K4A9928.CR2/threads:8/real_time_stddev 5 ms 5 ms 1 CPUTime,s=4.945m CPUTime/WallTime=46.3459u Pixels=0 Pixels/CPUTime=431.997k Pixels/WallTime=432.061k Raws/CPUTime=0.015432 Raws/WallTime=0.0154343 WallTime,s=4.94686m 2K4A9929.CR2/threads:8/real_time 306 ms 306 ms 2 CPUTime,s=0.306476 CPUTime/WallTime=0.999961 Pixels=12.4416M Pixels/CPUTime=40.5957M Pixels/WallTime=40.5941M Raws/CPUTime=3.2629 Raws/WallTime=3.26277 WallTime,s=0.306488 2K4A9929.CR2/threads:8/real_time 310 ms 310 ms 2 CPUTime,s=0.309978 CPUTime/WallTime=0.999939 Pixels=12.4416M Pixels/CPUTime=40.1371M Pixels/WallTime=40.1347M Raws/CPUTime=3.22604 Raws/WallTime=3.22584 WallTime,s=0.309996 2K4A9929.CR2/threads:8/real_time 309 ms 309 ms 2 CPUTime,s=0.30943 CPUTime/WallTime=0.999987 Pixels=12.4416M Pixels/CPUTime=40.2081M Pixels/WallTime=40.2076M Raws/CPUTime=3.23175 Raws/WallTime=3.23171 WallTime,s=0.309434 2K4A9929.CR2/threads:8/real_time_mean 309 ms 309 ms 2 CPUTime,s=0.308628 CPUTime/WallTime=0.999962 Pixels=12.4416M Pixels/CPUTime=40.3136M Pixels/WallTime=40.3121M Raws/CPUTime=3.24023 Raws/WallTime=3.24011 WallTime,s=0.308639 2K4A9929.CR2/threads:8/real_time_median 309 ms 309 ms 2 CPUTime,s=0.30943 CPUTime/WallTime=0.999961 Pixels=12.4416M Pixels/CPUTime=40.2081M Pixels/WallTime=40.2076M Raws/CPUTime=3.23175 Raws/WallTime=3.23171 WallTime,s=0.309434 2K4A9929.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=1.88354m CPUTime/WallTime=23.7788u Pixels=0 Pixels/CPUTime=246.821k Pixels/WallTime=246.914k Raws/CPUTime=0.0198384 Raws/WallTime=0.0198458 WallTime,s=1.88442m RUNNING: /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_out=/tmp/tmpRShmmf 2018-09-12 20:23:55 Running /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench Run on (8 X 4000 MHz CPU s) CPU Caches: L1 Data 16K (x8) L1 Instruction 64K (x4) L2 Unified 2048K (x4) L3 Unified 8192K (x1) ----------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... ----------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time 446 ms 446 ms 2 CPUTime,s=0.445589 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.19M Pixels/WallTime=118.19M Raws/CPUTime=2.24422 Raws/WallTime=2.24422 WallTime,s=0.445589 2K4A9927.CR2/threads:8/real_time 446 ms 446 ms 2 CPUTime,s=0.446008 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.079M Pixels/WallTime=118.08M Raws/CPUTime=2.24211 Raws/WallTime=2.24213 WallTime,s=0.446005 2K4A9927.CR2/threads:8/real_time 448 ms 448 ms 2 CPUTime,s=0.447763 CPUTime/WallTime=0.999994 Pixels=52.6643M Pixels/CPUTime=117.616M Pixels/WallTime=117.616M Raws/CPUTime=2.23332 Raws/WallTime=2.23331 WallTime,s=0.447766 2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446453 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=117.962M Pixels/WallTime=117.962M Raws/CPUTime=2.23988 Raws/WallTime=2.23989 WallTime,s=0.446453 2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.446008 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.079M Pixels/WallTime=118.08M Raws/CPUTime=2.24211 Raws/WallTime=2.24213 WallTime,s=0.446005 2K4A9927.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=1.15367m CPUTime/WallTime=6.48501u Pixels=0 Pixels/CPUTime=304.437k Pixels/WallTime=305.025k Raws/CPUTime=5.7807m Raws/WallTime=5.79188m WallTime,s=1.15591m 2K4A9928.CR2/threads:8/real_time 561 ms 561 ms 1 CPUTime,s=0.560856 CPUTime/WallTime=0.999996 Pixels=27.9936M Pixels/CPUTime=49.9123M Pixels/WallTime=49.9121M Raws/CPUTime=1.78299 Raws/WallTime=1.78298 WallTime,s=0.560858 2K4A9928.CR2/threads:8/real_time 560 ms 560 ms 1 CPUTime,s=0.560023 CPUTime/WallTime=1.00001 Pixels=27.9936M Pixels/CPUTime=49.9865M Pixels/WallTime=49.9872M Raws/CPUTime=1.78564 Raws/WallTime=1.78567 WallTime,s=0.560015 2K4A9928.CR2/threads:8/real_time 562 ms 562 ms 1 CPUTime,s=0.562337 CPUTime/WallTime=0.999994 Pixels=27.9936M Pixels/CPUTime=49.7808M Pixels/WallTime=49.7805M Raws/CPUTime=1.77829 Raws/WallTime=1.77828 WallTime,s=0.56234 2K4A9928.CR2/threads:8/real_time_mean 561 ms 561 ms 1 CPUTime,s=0.561072 CPUTime/WallTime=1 Pixels=27.9936M Pixels/CPUTime=49.8932M Pixels/WallTime=49.8933M Raws/CPUTime=1.78231 Raws/WallTime=1.78231 WallTime,s=0.561071 2K4A9928.CR2/threads:8/real_time_median 561 ms 561 ms 1 CPUTime,s=0.560856 CPUTime/WallTime=0.999996 Pixels=27.9936M Pixels/CPUTime=49.9123M Pixels/WallTime=49.9121M Raws/CPUTime=1.78299 Raws/WallTime=1.78298 WallTime,s=0.560858 2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=1.17202m CPUTime/WallTime=10.7929u Pixels=0 Pixels/CPUTime=104.164k Pixels/WallTime=104.612k Raws/CPUTime=3.721m Raws/WallTime=3.73701m WallTime,s=1.17706m 2K4A9929.CR2/threads:8/real_time 305 ms 305 ms 2 CPUTime,s=0.305436 CPUTime/WallTime=0.999926 Pixels=12.4416M Pixels/CPUTime=40.7339M Pixels/WallTime=40.7309M Raws/CPUTime=3.27401 Raws/WallTime=3.27376 WallTime,s=0.305459 2K4A9929.CR2/threads:8/real_time 306 ms 306 ms 2 CPUTime,s=0.30576 CPUTime/WallTime=0.999999 Pixels=12.4416M Pixels/CPUTime=40.6908M Pixels/WallTime=40.6908M Raws/CPUTime=3.27054 Raws/WallTime=3.27054 WallTime,s=0.30576 2K4A9929.CR2/threads:8/real_time 307 ms 307 ms 2 CPUTime,s=0.30724 CPUTime/WallTime=0.999991 Pixels=12.4416M Pixels/CPUTime=40.4947M Pixels/WallTime=40.4944M Raws/CPUTime=3.25478 Raws/WallTime=3.25475 WallTime,s=0.307243 2K4A9929.CR2/threads:8/real_time_mean 306 ms 306 ms 2 CPUTime,s=0.306145 CPUTime/WallTime=0.999972 Pixels=12.4416M Pixels/CPUTime=40.6398M Pixels/WallTime=40.6387M Raws/CPUTime=3.26645 Raws/WallTime=3.26635 WallTime,s=0.306154 2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.30576 CPUTime/WallTime=0.999991 Pixels=12.4416M Pixels/CPUTime=40.6908M Pixels/WallTime=40.6908M Raws/CPUTime=3.27054 Raws/WallTime=3.27054 WallTime,s=0.30576 2K4A9929.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=961.851u CPUTime/WallTime=40.2708u Pixels=0 Pixels/CPUTime=127.481k Pixels/WallTime=126.577k Raws/CPUTime=0.0102463 Raws/WallTime=0.0101737 WallTime,s=955.105u Comparing /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench to /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time -0.0038 -0.0038 447 446 447 446 2K4A9927.CR2/threads:8/real_time +0.0031 +0.0031 444 446 444 446 2K4A9927.CR2/threads:8/real_time -0.0031 -0.0031 447 446 447 446 2K4A9927.CR2/threads:8/real_time_pvalue 0.6428 0.6428 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9927.CR2/threads:8/real_time_mean +0.0006 +0.0006 446 446 446 446 2K4A9927.CR2/threads:8/real_time_median -0.0022 -0.0022 447 446 447 446 2K4A9927.CR2/threads:8/real_time_stddev -0.3161 -0.3175 2 1 2 1 2K4A9928.CR2/threads:8/real_time -0.0031 -0.0029 563 561 563 561 2K4A9928.CR2/threads:8/real_time -0.0009 -0.0008 561 561 561 561 2K4A9928.CR2/threads:8/real_time -0.0169 -0.0168 570 561 570 561 2K4A9928.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9928.CR2/threads:8/real_time_mean -0.0066 -0.0065 565 561 565 561 2K4A9928.CR2/threads:8/real_time_median -0.0031 -0.0029 563 561 563 561 2K4A9928.CR2/threads:8/real_time_stddev -0.7620 -0.7630 5 1 5 1 2K4A9929.CR2/threads:8/real_time -0.0034 -0.0034 306 305 306 305 2K4A9929.CR2/threads:8/real_time -0.0146 -0.0146 310 305 310 305 2K4A9929.CR2/threads:8/real_time -0.0128 -0.0129 309 305 309 305 2K4A9929.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9929.CR2/threads:8/real_time_mean -0.0081 -0.0080 309 306 309 306 2K4A9929.CR2/threads:8/real_time_median -0.0119 -0.0119 309 306 309 306 2K4A9929.CR2/threads:8/real_time_stddev -0.4931 -0.4894 2 1 2 1 $ ~/src/googlebenchmark/tools/compare.py -a benchmarks ~/rawspeed/build-{0,1}/src/utilities/rsbench/rsbench --benchmark_repetitions=3 * RUNNING: /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_display_aggregates_only=true --benchmark_out=/tmp/tmpjrD2I0 2018-09-12 20:24:11 Running /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench Run on (8 X 4000 MHz CPU s) CPU Caches: L1 Data 16K (x8) L1 Instruction 64K (x4) L2 Unified 2048K (x4) L3 Unified 8192K (x1) ----------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... ----------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.446077 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.061M Pixels/WallTime=118.062M Raws/CPUTime=2.24177 Raws/WallTime=2.24178 WallTime,s=0.446076 2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.445676 CPUTime/WallTime=1 Pixels=52.6643M Pixels/CPUTime=118.167M Pixels/WallTime=118.168M Raws/CPUTime=2.24378 Raws/WallTime=2.2438 WallTime,s=0.445673 2K4A9927.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=820.275u CPUTime/WallTime=3.51513u Pixels=0 Pixels/CPUTime=216.876k Pixels/WallTime=217.229k Raws/CPUTime=4.11809m Raws/WallTime=4.12478m WallTime,s=821.607u 2K4A9928.CR2/threads:8/real_time_mean 562 ms 562 ms 1 CPUTime,s=0.561843 CPUTime/WallTime=0.999983 Pixels=27.9936M Pixels/CPUTime=49.8247M Pixels/WallTime=49.8238M Raws/CPUTime=1.77986 Raws/WallTime=1.77983 WallTime,s=0.561852 2K4A9928.CR2/threads:8/real_time_median 562 ms 562 ms 1 CPUTime,s=0.561745 CPUTime/WallTime=0.999995 Pixels=27.9936M Pixels/CPUTime=49.8333M Pixels/WallTime=49.8307M Raws/CPUTime=1.78017 Raws/WallTime=1.78008 WallTime,s=0.561774 2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=788.58u CPUTime/WallTime=30.2578u Pixels=0 Pixels/CPUTime=69.914k Pixels/WallTime=69.4978k Raws/CPUTime=2.4975m Raws/WallTime=2.48263m WallTime,s=783.873u 2K4A9929.CR2/threads:8/real_time_mean 306 ms 306 ms 2 CPUTime,s=0.305718 CPUTime/WallTime=1.00001 Pixels=12.4416M Pixels/CPUTime=40.6964M Pixels/WallTime=40.6967M Raws/CPUTime=3.271 Raws/WallTime=3.27102 WallTime,s=0.305716 2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.305737 CPUTime/WallTime=1 Pixels=12.4416M Pixels/CPUTime=40.6939M Pixels/WallTime=40.6945M Raws/CPUTime=3.27079 Raws/WallTime=3.27084 WallTime,s=0.305732 2K4A9929.CR2/threads:8/real_time_stddev 1 ms 1 ms 2 CPUTime,s=584.969u CPUTime/WallTime=7.87539u Pixels=0 Pixels/CPUTime=77.8769k Pixels/WallTime=77.9118k Raws/CPUTime=6.2594m Raws/WallTime=6.2622m WallTime,s=585.232u RUNNING: /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench --benchmark_repetitions=3 2K4A9927.CR2 2K4A9928.CR2 2K4A9929.CR2 --benchmark_display_aggregates_only=true --benchmark_out=/tmp/tmpN9Xhk2 2018-09-12 20:24:18 Running /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench Run on (8 X 4000 MHz CPU s) CPU Caches: L1 Data 16K (x8) L1 Instruction 64K (x4) L2 Unified 2048K (x4) L3 Unified 8192K (x1) ----------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... ----------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time_mean 446 ms 446 ms 2 CPUTime,s=0.445771 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.142M Pixels/WallTime=118.143M Raws/CPUTime=2.2433 Raws/WallTime=2.24332 WallTime,s=0.445769 2K4A9927.CR2/threads:8/real_time_median 446 ms 446 ms 2 CPUTime,s=0.445719 CPUTime/WallTime=1.00001 Pixels=52.6643M Pixels/CPUTime=118.156M Pixels/WallTime=118.156M Raws/CPUTime=2.24356 Raws/WallTime=2.24358 WallTime,s=0.445717 2K4A9927.CR2/threads:8/real_time_stddev 0 ms 0 ms 2 CPUTime,s=184.23u CPUTime/WallTime=1.69441u Pixels=0 Pixels/CPUTime=48.8185k Pixels/WallTime=49.0086k Raws/CPUTime=926.975u Raws/WallTime=930.585u WallTime,s=184.946u 2K4A9928.CR2/threads:8/real_time_mean 562 ms 562 ms 1 CPUTime,s=0.562071 CPUTime/WallTime=0.999969 Pixels=27.9936M Pixels/CPUTime=49.8045M Pixels/WallTime=49.803M Raws/CPUTime=1.77914 Raws/WallTime=1.77908 WallTime,s=0.562088 2K4A9928.CR2/threads:8/real_time_median 562 ms 562 ms 1 CPUTime,s=0.56237 CPUTime/WallTime=0.999986 Pixels=27.9936M Pixels/CPUTime=49.7779M Pixels/WallTime=49.7772M Raws/CPUTime=1.77819 Raws/WallTime=1.77816 WallTime,s=0.562378 2K4A9928.CR2/threads:8/real_time_stddev 1 ms 1 ms 1 CPUTime,s=967.38u CPUTime/WallTime=35.0024u Pixels=0 Pixels/CPUTime=85.7806k Pixels/WallTime=84.0545k Raws/CPUTime=3.06429m Raws/WallTime=3.00263m WallTime,s=947.993u 2K4A9929.CR2/threads:8/real_time_mean 307 ms 307 ms 2 CPUTime,s=0.306511 CPUTime/WallTime=0.999995 Pixels=12.4416M Pixels/CPUTime=40.5924M Pixels/WallTime=40.5922M Raws/CPUTime=3.26264 Raws/WallTime=3.26262 WallTime,s=0.306513 2K4A9929.CR2/threads:8/real_time_median 306 ms 306 ms 2 CPUTime,s=0.306169 CPUTime/WallTime=0.999999 Pixels=12.4416M Pixels/CPUTime=40.6364M Pixels/WallTime=40.637M Raws/CPUTime=3.26618 Raws/WallTime=3.26622 WallTime,s=0.306164 2K4A9929.CR2/threads:8/real_time_stddev 2 ms 2 ms 2 CPUTime,s=2.25763m CPUTime/WallTime=21.4306u Pixels=0 Pixels/CPUTime=298.503k Pixels/WallTime=299.126k Raws/CPUTime=0.0239924 Raws/WallTime=0.0240424 WallTime,s=2.26242m Comparing /home/lebedevri/rawspeed/build-0/src/utilities/rsbench/rsbench to /home/lebedevri/rawspeed/build-1/src/utilities/rsbench/rsbench Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------------------------- 2K4A9927.CR2/threads:8/real_time_pvalue 0.6428 0.6428 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9927.CR2/threads:8/real_time_mean -0.0007 -0.0007 446 446 446 446 2K4A9927.CR2/threads:8/real_time_median +0.0001 +0.0001 446 446 446 446 2K4A9927.CR2/threads:8/real_time_stddev -0.7746 -0.7749 1 0 1 0 2K4A9928.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9928.CR2/threads:8/real_time_mean +0.0004 +0.0004 562 562 562 562 2K4A9928.CR2/threads:8/real_time_median +0.0011 +0.0011 562 562 562 562 2K4A9928.CR2/threads:8/real_time_stddev +0.2091 +0.2270 1 1 1 1 2K4A9929.CR2/threads:8/real_time_pvalue 0.0636 0.0636 U Test, Repetitions: 3. WARNING: Results unreliable! 9+ repetitions recommended. 2K4A9929.CR2/threads:8/real_time_mean +0.0026 +0.0026 306 307 306 307 2K4A9929.CR2/threads:8/real_time_median +0.0014 +0.0014 306 306 306 306 2K4A9929.CR2/threads:8/real_time_stddev +2.8652 +2.8585 1 2 1 2 ``` --- docs/tools.md | 9 +++++ tools/compare.py | 41 +++++++++++++++++++- tools/gbench/Inputs/test3_run0.json | 10 +++++ tools/gbench/Inputs/test3_run1.json | 10 +++++ tools/gbench/report.py | 60 ++++++++++++++++++++++++++++- 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/docs/tools.md b/docs/tools.md index 73ba27852e..4a3b2e9bd2 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -6,6 +6,15 @@ The `compare.py` can be used to compare the result of benchmarks. **NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html). +### Displaying aggregates only + +The switch `-a` / `--display_aggregates_only` can be used to control the +displayment of the normal iterations vs the aggregates. When passed, it will +be passthrough to the benchmark binaries to be run, and will be accounted for +in the tool itself; only the aggregates will be displayed, but not normal runs. +It only affects the display, the separate runs will still be used to calculate +the U test. + ### Modes of operation There are three modes of operation: diff --git a/tools/compare.py b/tools/compare.py index d27e24b349..9ff5c1424d 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -36,6 +36,17 @@ def create_parser(): parser = ArgumentParser( description='versatile benchmark output compare tool') + parser.add_argument( + '-a', + '--display_aggregates_only', + dest='display_aggregates_only', + action="store_true", + help="If there are repetitions, by default, we display everything - the" + " actual runs, and the aggregates computed. Sometimes, it is " + "desirable to only view the aggregates. E.g. when there are a lot " + "of repetitions. Do note that only the display is affected. " + "Internally, all the actual runs are still used, e.g. for U test.") + utest = parser.add_argument_group() utest.add_argument( '--no-utest', @@ -200,6 +211,9 @@ def main(): check_inputs(test_baseline, test_contender, benchmark_options) + if args.display_aggregates_only: + benchmark_options += ['--benchmark_display_aggregates_only=true'] + options_baseline = [] options_contender = [] @@ -223,7 +237,8 @@ def main(): # Diff and output output_lines = gbench.report.generate_difference_report( - json1, json2, args.utest, args.utest_alpha) + json1, json2, args.display_aggregates_only, + args.utest, args.utest_alpha) print(description) for ln in output_lines: print(ln) @@ -246,6 +261,7 @@ def setUp(self): def test_benchmarks_basic(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -255,6 +271,7 @@ def test_benchmarks_basic(self): def test_benchmarks_basic_without_utest(self): parsed = self.parser.parse_args( ['--no-utest', 'benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.display_aggregates_only) self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.05) self.assertEqual(parsed.mode, 'benchmarks') @@ -262,9 +279,20 @@ def test_benchmarks_basic_without_utest(self): self.assertEqual(parsed.test_contender[0].name, self.testInput1) self.assertFalse(parsed.benchmark_options) + def test_benchmarks_basic_display_aggregates_only(self): + parsed = self.parser.parse_args( + ['-a', 'benchmarks', self.testInput0, self.testInput1]) + self.assertTrue(parsed.display_aggregates_only) + self.assertTrue(parsed.utest) + self.assertEqual(parsed.mode, 'benchmarks') + self.assertEqual(parsed.test_baseline[0].name, self.testInput0) + self.assertEqual(parsed.test_contender[0].name, self.testInput1) + self.assertFalse(parsed.benchmark_options) + def test_benchmarks_basic_with_utest_alpha(self): parsed = self.parser.parse_args( ['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.314) self.assertEqual(parsed.mode, 'benchmarks') @@ -275,6 +303,7 @@ def test_benchmarks_basic_with_utest_alpha(self): def test_benchmarks_basic_without_utest_with_utest_alpha(self): parsed = self.parser.parse_args( ['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1]) + self.assertFalse(parsed.display_aggregates_only) self.assertFalse(parsed.utest) self.assertEqual(parsed.utest_alpha, 0.314) self.assertEqual(parsed.mode, 'benchmarks') @@ -285,6 +314,7 @@ def test_benchmarks_basic_without_utest_with_utest_alpha(self): def test_benchmarks_with_remainder(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, 'd']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -294,6 +324,7 @@ def test_benchmarks_with_remainder(self): def test_benchmarks_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarks', self.testInput0, self.testInput1, '--', 'e']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarks') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -303,6 +334,7 @@ def test_benchmarks_with_remainder_after_doubleminus(self): def test_filters_basic(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) @@ -313,6 +345,7 @@ def test_filters_basic(self): def test_filters_with_remainder(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', 'e']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) @@ -323,6 +356,7 @@ def test_filters_with_remainder(self): def test_filters_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['filters', self.testInput0, 'c', 'd', '--', 'f']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'filters') self.assertEqual(parsed.test[0].name, self.testInput0) @@ -333,6 +367,7 @@ def test_filters_with_remainder_after_doubleminus(self): def test_benchmarksfiltered_basic(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -344,6 +379,7 @@ def test_benchmarksfiltered_basic(self): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -355,6 +391,7 @@ def test_benchmarksfiltered_with_remainder(self): def test_benchmarksfiltered_with_remainder_after_doubleminus(self): parsed = self.parser.parse_args( ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g']) + self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, 'benchmarksfiltered') self.assertEqual(parsed.test_baseline[0].name, self.testInput0) @@ -365,7 +402,7 @@ def test_benchmarksfiltered_with_remainder_after_doubleminus(self): if __name__ == '__main__': - # unittest.main() + #unittest.main() main() # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/tools/gbench/Inputs/test3_run0.json b/tools/gbench/Inputs/test3_run0.json index ca793f3367..60e83273b8 100644 --- a/tools/gbench/Inputs/test3_run0.json +++ b/tools/gbench/Inputs/test3_run0.json @@ -9,6 +9,7 @@ "benchmarks": [ { "name": "BM_One", + "run_type": "aggregate", "iterations": 1000, "real_time": 10, "cpu_time": 100, @@ -30,6 +31,15 @@ }, { "name": "short", + "run_type": "aggregate", + "iterations": 1000, + "real_time": 8, + "cpu_time": 80, + "time_unit": "ns" + }, + { + "name": "medium", + "run_type": "iteration", "iterations": 1000, "real_time": 8, "cpu_time": 80, diff --git a/tools/gbench/Inputs/test3_run1.json b/tools/gbench/Inputs/test3_run1.json index e5cf50c744..c6faa6bc39 100644 --- a/tools/gbench/Inputs/test3_run1.json +++ b/tools/gbench/Inputs/test3_run1.json @@ -16,6 +16,7 @@ }, { "name": "BM_Two", + "run_type": "aggregate", "iterations": 1000, "real_time": 10, "cpu_time": 89, @@ -30,10 +31,19 @@ }, { "name": "short", + "run_type": "aggregate", "iterations": 1000, "real_time": 8, "cpu_time": 80, "time_unit": "ns" + }, + { + "name": "medium", + "run_type": "iteration", + "iterations": 1200, + "real_time": 5, + "cpu_time": 53, + "time_unit": "ns" } ] } diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 4d03a54767..7696d30ed6 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -96,6 +96,7 @@ def filter_benchmark(json_orig, family, replacement=""): def generate_difference_report( json1, json2, + display_aggregates_only=False, utest=False, utest_alpha=0.05, use_color=True): @@ -191,6 +192,13 @@ def get_color(res): timings_cpu[0].append(bn['cpu_time']) timings_cpu[1].append(other_bench['cpu_time']) + # *After* recording this run for u test, *if* we were asked to only + # display aggregates, and if it is non-aggregate, then skip it. + if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench: + assert bn['run_type'] == other_bench['run_type'] + if bn['run_type'] != 'aggregate': + continue; + tres = calculate_change(timings_time[0][-1], timings_time[1][-1]) cpures = calculate_change(timings_cpu[0][-1], timings_cpu[1][-1]) output_strs += [color_format(use_color, @@ -311,6 +319,55 @@ def load_results(self): json2 = json.load(f) return json1, json2 + def test_utest(self): + expect_lines = [] + expect_lines = [ + ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], + ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], + ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'], + ['BM_Two_pvalue', + '0.2207', + '0.6831', + 'U', + 'Test,', + 'Repetitions:', + '2.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], + ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'], + ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], + ] + json1, json2 = self.load_results() + output_lines_with_header = generate_difference_report( + json1, json2, utest=True, utest_alpha=0.05, use_color=False) + output_lines = output_lines_with_header[2:] + print("\n") + print("\n".join(output_lines_with_header)) + self.assertEqual(len(output_lines), len(expect_lines)) + for i in range(0, len(output_lines)): + parts = [x for x in output_lines[i].split(' ') if x] + self.assertEqual(parts, expect_lines[i]) + + +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase): + def load_results(self): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput1 = os.path.join(testInputs, 'test3_run0.json') + testOutput2 = os.path.join(testInputs, 'test3_run1.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + def test_utest(self): expect_lines = [] expect_lines = [ @@ -334,7 +391,8 @@ def test_utest(self): ] json1, json2 = self.load_results() output_lines_with_header = generate_difference_report( - json1, json2, True, 0.05, use_color=False) + json1, json2, display_aggregates_only=True, + utest=True, utest_alpha=0.05, use_color=False) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) From 52613079824ac58d06c070aa9fbbb186a5859e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Tue, 18 Sep 2018 11:42:20 +0300 Subject: [PATCH 087/330] [benchmark] Lowercase windows specific includes (#679) The windows SDK headers don't have self-consistent casing anyway, and many projects consistently use lowercase for them, in order to fix crosscompilation with mingw headers. --- src/colorprint.cc | 2 +- src/sleep.cc | 2 +- src/sysinfo.cc | 6 +++--- src/timers.cc | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/colorprint.cc b/src/colorprint.cc index 2dec4a8b28..fff6a98818 100644 --- a/src/colorprint.cc +++ b/src/colorprint.cc @@ -25,7 +25,7 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS -#include +#include #include #else #include diff --git a/src/sleep.cc b/src/sleep.cc index 54aa04a422..1512ac90f7 100644 --- a/src/sleep.cc +++ b/src/sleep.cc @@ -21,7 +21,7 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS -#include +#include #endif namespace benchmark { diff --git a/src/sysinfo.cc b/src/sysinfo.cc index f79412b542..d4e5336fdb 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -15,10 +15,10 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS -#include +#include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA -#include -#include +#include +#include #else #include #ifndef BENCHMARK_OS_FUCHSIA diff --git a/src/timers.cc b/src/timers.cc index 2010e2450b..7613ff92c6 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -16,10 +16,10 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS -#include +#include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA -#include -#include +#include +#include #else #include #ifndef BENCHMARK_OS_FUCHSIA From 439d6b1c2a6da5cb6adc4c4dfc555af235722396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 19 Sep 2018 13:52:05 +0300 Subject: [PATCH 088/330] Include sys/time.h for cycleclock.h when building on MinGW (#680) When building for ARM, there is a fallback codepath that uses gettimeofday, which requires sys/time.h. The Windows SDK doesn't have this header, but MinGW does have it. Thus, this fixes building for Windows on ARM with MinGW headers/libraries, while Windows on ARM with the Windows SDK still is broken. --- src/cycleclock.h | 2 +- src/internal_macros.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cycleclock.h b/src/cycleclock.h index 00d5764167..f5e37b011b 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -41,7 +41,7 @@ extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif -#ifndef BENCHMARK_OS_WINDOWS +#if !defined(BENCHMARK_OS_WINDOWS) || defined(BENCHMARK_OS_MINGW) #include #include #endif diff --git a/src/internal_macros.h b/src/internal_macros.h index 32089e69f6..5dbf4fd275 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -40,6 +40,9 @@ #define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 + #if defined(__MINGW32__) + #define BENCHMARK_OS_MINGW 1 + #endif #elif defined(__APPLE__) #define BENCHMARK_OS_APPLE 1 #include "TargetConditionals.h" From aad33aab3c1a8db06053665af39e4679bc774b74 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 19 Sep 2018 15:59:13 +0300 Subject: [PATCH 089/330] [Tooling] Rewrite generate_difference_report(). (#678) My knowledge of python is not great, so this is kinda horrible. Two things: 1. If there were repetitions, for the RHS (i.e. the new value) we were always using the first repetition, which naturally results in incorrect change reports for the second and following repetitions. And what is even worse, that completely broke U test. :( 2. A better support for different repetition count for U test was missing. It's important if we are to be able to report 'iteration as repetition', since it is rather likely that the iteration count will mismatch. Now, the rough idea on how this is implemented now. I think this is the right solution. 1. Get all benchmark names (in order) from the lhs benchmark. 2. While preserving the order, keep the unique names 3. Get all benchmark names (in order) from the rhs benchmark. 4. While preserving the order, keep the unique names 5. Intersect `2.` and `4.`, get the list of unique benchmark names that exist on both sides. 6. Now, we want to group (partition) all the benchmarks with the same name. ``` BM_FOO: [lhs]: BM_FOO/repetition0 BM_FOO/repetition1 [rhs]: BM_FOO/repetition0 BM_FOO/repetition1 BM_FOO/repetition2 ... ``` We also drop mismatches in `time_unit` here. _(whose bright idea was it to store arbitrarily scaled timers in json **?!** )_ 7. Iterate for each partition 7.1. Conditionally, diff the overlapping repetitions (the count of repetitions may be different.) 7.2. Conditionally, do the U test: 7.2.1. Get **all** the values of `"real_time"` field from the lhs benchmark 7.2.2. Get **all** the values of `"cpu_time"` field from the lhs benchmark 7.2.3. Get **all** the values of `"real_time"` field from the rhs benchmark 7.2.4. Get **all** the values of `"cpu_time"` field from the rhs benchmark NOTE: the repetition count may be different, but we want *all* the values! 7.2.5. Do the rest of the u test stuff 7.2.6. Print u test 8. ??? 9. **PROFIT**! Fixes #677 --- tools/gbench/Inputs/test3_run0.json | 18 +- tools/gbench/Inputs/test3_run1.json | 22 +- tools/gbench/report.py | 327 +++++++++++++++++++--------- 3 files changed, 256 insertions(+), 111 deletions(-) diff --git a/tools/gbench/Inputs/test3_run0.json b/tools/gbench/Inputs/test3_run0.json index 60e83273b8..49f8b06143 100644 --- a/tools/gbench/Inputs/test3_run0.json +++ b/tools/gbench/Inputs/test3_run0.json @@ -26,7 +26,7 @@ "name": "BM_Two", "iterations": 1000, "real_time": 8, - "cpu_time": 80, + "cpu_time": 86, "time_unit": "ns" }, { @@ -37,6 +37,14 @@ "cpu_time": 80, "time_unit": "ns" }, + { + "name": "short", + "run_type": "aggregate", + "iterations": 1000, + "real_time": 8, + "cpu_time": 77, + "time_unit": "ns" + }, { "name": "medium", "run_type": "iteration", @@ -44,6 +52,14 @@ "real_time": 8, "cpu_time": 80, "time_unit": "ns" + }, + { + "name": "medium", + "run_type": "iteration", + "iterations": 1000, + "real_time": 9, + "cpu_time": 82, + "time_unit": "ns" } ] } diff --git a/tools/gbench/Inputs/test3_run1.json b/tools/gbench/Inputs/test3_run1.json index c6faa6bc39..acc5ba17ae 100644 --- a/tools/gbench/Inputs/test3_run1.json +++ b/tools/gbench/Inputs/test3_run1.json @@ -26,15 +26,31 @@ "name": "BM_Two", "iterations": 1000, "real_time": 7, - "cpu_time": 70, + "cpu_time": 72, "time_unit": "ns" }, { "name": "short", "run_type": "aggregate", "iterations": 1000, - "real_time": 8, - "cpu_time": 80, + "real_time": 7, + "cpu_time": 75, + "time_unit": "ns" + }, + { + "name": "short", + "run_type": "aggregate", + "iterations": 762, + "real_time": 4.54, + "cpu_time": 66.6, + "time_unit": "ns" + }, + { + "name": "short", + "run_type": "iteration", + "iterations": 1000, + "real_time": 800, + "cpu_time": 1, "time_unit": "ns" }, { diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 7696d30ed6..28bab34af7 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -36,6 +36,7 @@ def __format__(self, format): UTEST_MIN_REPETITIONS = 2 UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. +UTEST_COL_NAME = "_pvalue" def color_format(use_color, fmt_str, *args, **kwargs): @@ -93,6 +94,99 @@ def filter_benchmark(json_orig, family, replacement=""): return filtered +def get_unique_benchmark_names(json): + """ + While *keeping* the order, give all the unique 'names' used for benchmarks. + """ + seen = set() + uniqued = [x['name'] for x in json['benchmarks'] + if x['name'] not in seen and + (seen.add(x['name']) or True)] + return uniqued + + +def intersect(list1, list2): + """ + Given two lists, get a new list consisting of the elements only contained + in *both of the input lists*, while preserving the ordering. + """ + return [x for x in list1 if x in list2] + + +def partition_benchmarks(json1, json2): + """ + While preserving the ordering, find benchmarks with the same names in + both of the inputs, and group them. + (i.e. partition/filter into groups with common name) + """ + json1_unique_names = get_unique_benchmark_names(json1) + json2_unique_names = get_unique_benchmark_names(json2) + names = intersect(json1_unique_names, json2_unique_names) + partitions = [] + for name in names: + # Pick the time unit from the first entry of the lhs benchmark. + time_unit = (x['time_unit'] + for x in json1['benchmarks'] if x['name'] == name).next() + # Filter by name and time unit. + lhs = [x for x in json1['benchmarks'] if x['name'] == name and + x['time_unit'] == time_unit] + rhs = [x for x in json2['benchmarks'] if x['name'] == name and + x['time_unit'] == time_unit] + partitions.append([lhs, rhs]) + return partitions + + +def extract_field(partition, field_name): + # The count of elements may be different. We want *all* of them. + lhs = [x[field_name] for x in partition[0]] + rhs = [x[field_name] for x in partition[1]] + return [lhs, rhs] + + +def print_utest(partition, utest_alpha, first_col_width, use_color=True): + timings_time = extract_field(partition, 'real_time') + timings_cpu = extract_field(partition, 'cpu_time') + + min_rep_cnt = min(len(timings_time[0]), + len(timings_time[1]), + len(timings_cpu[0]), + len(timings_cpu[1])) + + # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions? + if min_rep_cnt < UTEST_MIN_REPETITIONS: + return [] + + def get_utest_color(pval): + return BC_FAIL if pval >= utest_alpha else BC_OKGREEN + + time_pvalue = mannwhitneyu( + timings_time[0], timings_time[1], alternative='two-sided').pvalue + cpu_pvalue = mannwhitneyu( + timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue + + dsc = "U Test, Repetitions: {} vs {}".format( + len(timings_cpu[0]), len(timings_cpu[1])) + dsc_color = BC_OKGREEN + + if min_rep_cnt < UTEST_OPTIMAL_REPETITIONS: + dsc_color = BC_WARNING + dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( + UTEST_OPTIMAL_REPETITIONS) + + special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}" + + last_name = partition[0][0]['name'] + return [color_format(use_color, + special_str, + BC_HEADER, + "{}{}".format(last_name, UTEST_COL_NAME), + first_col_width, + get_utest_color(time_pvalue), time_pvalue, + get_utest_color(cpu_pvalue), cpu_pvalue, + dsc_color, dsc, + endc=BC_ENDC)] + + def generate_difference_report( json1, json2, @@ -113,110 +207,65 @@ def find_test(name): return b return None - utest_col_name = "_pvalue" first_col_width = max( first_col_width, len('Benchmark')) - first_col_width += len(utest_col_name) + first_col_width += len(UTEST_COL_NAME) first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format( 'Benchmark', 12 + first_col_width) output_strs = [first_line, '-' * len(first_line)] - last_name = None - timings_time = [[], []] - timings_cpu = [[], []] - - gen = (bn for bn in json1['benchmarks'] - if 'real_time' in bn and 'cpu_time' in bn) - for bn in gen: - fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" - special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}" - - if last_name is None: - last_name = bn['name'] - if last_name != bn['name']: - if ((len(timings_time[0]) >= UTEST_MIN_REPETITIONS) and - (len(timings_time[1]) >= UTEST_MIN_REPETITIONS) and - (len(timings_cpu[0]) >= UTEST_MIN_REPETITIONS) and - (len(timings_cpu[1]) >= UTEST_MIN_REPETITIONS)): - if utest: - def get_utest_color(pval): - if pval >= utest_alpha: - return BC_FAIL - else: - return BC_OKGREEN - time_pvalue = mannwhitneyu( - timings_time[0], timings_time[1], alternative='two-sided').pvalue - cpu_pvalue = mannwhitneyu( - timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue - dsc = "U Test, Repetitions: {}".format(len(timings_cpu[0])) - dsc_color = BC_OKGREEN - if len(timings_cpu[0]) < UTEST_OPTIMAL_REPETITIONS: - dsc_color = BC_WARNING - dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( - UTEST_OPTIMAL_REPETITIONS) - output_strs += [color_format(use_color, - special_str, - BC_HEADER, - "{}{}".format(last_name, - utest_col_name), - first_col_width, - get_utest_color(time_pvalue), - time_pvalue, - get_utest_color(cpu_pvalue), - cpu_pvalue, - dsc_color, - dsc, - endc=BC_ENDC)] - last_name = bn['name'] - timings_time = [[], []] - timings_cpu = [[], []] - - other_bench = find_test(bn['name']) - if not other_bench: - continue - - if bn['time_unit'] != other_bench['time_unit']: - continue + partitions = partition_benchmarks(json1, json2) + for partition in partitions: + # Careful, we may have different repetition count. + for i in range(min(len(partition[0]), len(partition[1]))): + bn = partition[0][i] + other_bench = partition[1][i] + + # *If* we were asked to only display aggregates, + # and if it is non-aggregate, then skip it. + if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench: + assert bn['run_type'] == other_bench['run_type'] + if bn['run_type'] != 'aggregate': + continue + + fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" + + def get_color(res): + if res > 0.05: + return BC_FAIL + elif res > -0.07: + return BC_WHITE + else: + return BC_CYAN + + tres = calculate_change(bn['real_time'], other_bench['real_time']) + cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time']) + output_strs += [color_format(use_color, + fmt_str, + BC_HEADER, + bn['name'], + first_col_width, + get_color(tres), + tres, + get_color(cpures), + cpures, + bn['real_time'], + other_bench['real_time'], + bn['cpu_time'], + other_bench['cpu_time'], + endc=BC_ENDC)] + + # After processing the whole partition, if requested, do the U test. + if utest: + output_strs += print_utest(partition, + utest_alpha=utest_alpha, + first_col_width=first_col_width, + use_color=use_color) - def get_color(res): - if res > 0.05: - return BC_FAIL - elif res > -0.07: - return BC_WHITE - else: - return BC_CYAN - - timings_time[0].append(bn['real_time']) - timings_time[1].append(other_bench['real_time']) - timings_cpu[0].append(bn['cpu_time']) - timings_cpu[1].append(other_bench['cpu_time']) - - # *After* recording this run for u test, *if* we were asked to only - # display aggregates, and if it is non-aggregate, then skip it. - if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench: - assert bn['run_type'] == other_bench['run_type'] - if bn['run_type'] != 'aggregate': - continue; - - tres = calculate_change(timings_time[0][-1], timings_time[1][-1]) - cpures = calculate_change(timings_cpu[0][-1], timings_cpu[1][-1]) - output_strs += [color_format(use_color, - fmt_str, - BC_HEADER, - bn['name'], - first_col_width, - get_color(tres), - tres, - get_color(cpures), - cpures, - timings_time[0][-1], - timings_time[1][-1], - timings_cpu[0][-1], - timings_cpu[1][-1], - endc=BC_ENDC)] return output_strs + ############################################################################### # Unit tests @@ -224,6 +273,33 @@ def get_color(res): import unittest +class TestGetUniqueBenchmarkNames(unittest.TestCase): + def load_results(self): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput = os.path.join(testInputs, 'test3_run0.json') + with open(testOutput, 'r') as f: + json = json.load(f) + return json + + def test_basic(self): + expect_lines = [ + 'BM_One', + 'BM_Two', + 'short', # These two are not sorted + 'medium', # These two are not sorted + ] + json = self.load_results() + output_lines = get_unique_benchmark_names(json) + print("\n") + print("\n".join(output_lines)) + self.assertEqual(len(output_lines), len(expect_lines)) + for i in range(0, len(output_lines)): + self.assertEqual(expect_lines[i], output_lines[i]) + class TestReportDifference(unittest.TestCase): def load_results(self): import json @@ -267,7 +343,7 @@ def test_basic(self): for i in range(0, len(output_lines)): parts = [x for x in output_lines[i].split(' ') if x] self.assertEqual(len(parts), 7) - self.assertEqual(parts, expect_lines[i]) + self.assertEqual(expect_lines[i], parts) class TestReportDifferenceBetweenFamilies(unittest.TestCase): @@ -301,7 +377,7 @@ def test_basic(self): for i in range(0, len(output_lines)): parts = [x for x in output_lines[i].split(' ') if x] self.assertEqual(len(parts), 7) - self.assertEqual(parts, expect_lines[i]) + self.assertEqual(expect_lines[i], parts) class TestReportDifferenceWithUTest(unittest.TestCase): @@ -324,13 +400,15 @@ def test_utest(self): expect_lines = [ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], - ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'], + ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], ['BM_Two_pvalue', - '0.2207', - '0.6831', + '0.6985', + '0.6985', 'U', 'Test,', 'Repetitions:', + '2', + 'vs', '2.', 'WARNING:', 'Results', @@ -338,7 +416,23 @@ def test_utest(self): '9+', 'repetitions', 'recommended.'], - ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'], + ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], + ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], + ['short_pvalue', + '0.7671', + '0.1489', + 'U', + 'Test,', + 'Repetitions:', + '2', + 'vs', + '3.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], ] json1, json2 = self.load_results() @@ -350,10 +444,11 @@ def test_utest(self): self.assertEqual(len(output_lines), len(expect_lines)) for i in range(0, len(output_lines)): parts = [x for x in output_lines[i].split(' ') if x] - self.assertEqual(parts, expect_lines[i]) + self.assertEqual(expect_lines[i], parts) -class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase): +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( + unittest.TestCase): def load_results(self): import json testInputs = os.path.join( @@ -373,13 +468,15 @@ def test_utest(self): expect_lines = [ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], - ['BM_Two', '+0.2500', '+0.1125', '8', '10', '80', '89'], + ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], ['BM_Two_pvalue', - '0.2207', - '0.6831', + '0.6985', + '0.6985', 'U', 'Test,', 'Repetitions:', + '2', + 'vs', '2.', 'WARNING:', 'Results', @@ -387,7 +484,23 @@ def test_utest(self): '9+', 'repetitions', 'recommended.'], - ['short', '+0.0000', '+0.0000', '8', '8', '80', '80'], + ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], + ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], + ['short_pvalue', + '0.7671', + '0.1489', + 'U', + 'Test,', + 'Repetitions:', + '2', + 'vs', + '3.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], ] json1, json2 = self.load_results() output_lines_with_header = generate_difference_report( @@ -399,7 +512,7 @@ def test_utest(self): self.assertEqual(len(output_lines), len(expect_lines)) for i in range(0, len(output_lines)): parts = [x for x in output_lines[i].split(' ') if x] - self.assertEqual(parts, expect_lines[i]) + self.assertEqual(expect_lines[i], parts) if __name__ == '__main__': From eb8cbec0776455463274ea9947ab0ecfe0f768fe Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 26 Sep 2018 12:11:54 +0300 Subject: [PATCH 090/330] appveyor ci: drop Visual Studio 12 2013 - unsupported by Google Test master branch. (#691) See https://github.com/google/googletest/pull/1815 Fixes #689. --- appveyor.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e99c6e77f0..cf240190be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,12 +20,6 @@ environment: - compiler: msvc-14-seh generator: "Visual Studio 14 2015 Win64" - - compiler: msvc-12-seh - generator: "Visual Studio 12 2013" - - - compiler: msvc-12-seh - generator: "Visual Studio 12 2013 Win64" - - compiler: gcc-5.3.0-posix generator: "MinGW Makefiles" cxx_path: 'C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin' From edc77a3669026eddc380721d5a3cdccd752b76cb Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 28 Sep 2018 12:28:43 +0100 Subject: [PATCH 091/330] Make State constructor private. (#650) The State constructor should not be part of the public API. Adding a utility method to BenchmarkInstance allows us to avoid leaking the RunInThread method into the public API. --- include/benchmark/benchmark.h | 10 ++++------ src/benchmark.cc | 18 ++++++++---------- src/benchmark_api_internal.cc | 15 +++++++++++++++ src/benchmark_api_internal.h | 8 ++++++-- src/benchmark_register.cc | 8 ++++---- 5 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 src/benchmark_api_internal.cc diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 04fa202fdf..609dc863e3 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -430,6 +430,7 @@ struct Statistics { }; namespace internal { +struct BenchmarkInstance; class ThreadTimer; class ThreadManager; @@ -664,12 +665,11 @@ class State { // Number of threads concurrently executing the benchmark. const int threads; - // TODO(EricWF) make me private + private: State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager); - private: void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. @@ -677,7 +677,8 @@ class State { void FinishKeepRunning(); internal::ThreadTimer* timer_; internal::ThreadManager* manager_; - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(State); + + friend struct internal::BenchmarkInstance; }; inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { @@ -931,9 +932,6 @@ class Benchmark { virtual void Run(State& state) = 0; - // Used inside the benchmark implementation - struct Instance; - protected: explicit Benchmark(const char* name); Benchmark(Benchmark const&); diff --git a/src/benchmark.cc b/src/benchmark.cc index dcdd23f40e..ffa75889d5 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -126,7 +126,7 @@ void UseCharPointer(char const volatile*) {} namespace { BenchmarkReporter::Run CreateRunReport( - const benchmark::internal::Benchmark::Instance& b, + const benchmark::internal::BenchmarkInstance& b, const internal::ThreadManager::Result& results, size_t memory_iterations, const MemoryManager::Result& memory_result, double seconds) { // Create report about this benchmark run. @@ -169,12 +169,10 @@ BenchmarkReporter::Run CreateRunReport( // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. -void RunInThread(const benchmark::internal::Benchmark::Instance* b, - size_t iters, int thread_id, - internal::ThreadManager* manager) { +void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, + ThreadManager* manager) { internal::ThreadTimer timer; - State st(iters, b->arg, thread_id, b->threads, &timer, manager); - b->benchmark->Run(st); + State st = b->Run(iters, thread_id, &timer, manager); CHECK(st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { @@ -199,7 +197,7 @@ struct RunResults { }; RunResults RunBenchmark( - const benchmark::internal::Benchmark::Instance& b, + const benchmark::internal::BenchmarkInstance& b, std::vector* complexity_reports) { RunResults run_results; @@ -437,7 +435,7 @@ void State::FinishKeepRunning() { namespace internal { namespace { -void RunBenchmarks(const std::vector& benchmarks, +void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { // Note the file_reporter can be null. @@ -447,7 +445,7 @@ void RunBenchmarks(const std::vector& benchmarks, bool has_repetitions = FLAGS_benchmark_repetitions > 1; size_t name_field_width = 10; size_t stat_field_width = 0; - for (const Benchmark::Instance& benchmark : benchmarks) { + for (const BenchmarkInstance& benchmark : benchmarks) { name_field_width = std::max(name_field_width, benchmark.name.size()); has_repetitions |= benchmark.repetitions > 1; @@ -595,7 +593,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, file_reporter->SetErrorStream(&output_file); } - std::vector benchmarks; + std::vector benchmarks; if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; if (benchmarks.empty()) { diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc new file mode 100644 index 0000000000..8d3108363b --- /dev/null +++ b/src/benchmark_api_internal.cc @@ -0,0 +1,15 @@ +#include "benchmark_api_internal.h" + +namespace benchmark { +namespace internal { + +State BenchmarkInstance::Run( + size_t iters, int thread_id, internal::ThreadTimer* timer, + internal::ThreadManager* manager) const { + State st(iters, arg, thread_id, threads, timer, manager); + benchmark->Run(st); + return st; +} + +} // internal +} // benchmark diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 0bf0005e84..fe492c7eb6 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -13,7 +14,7 @@ namespace benchmark { namespace internal { // Information kept per benchmark we may want to run -struct Benchmark::Instance { +struct BenchmarkInstance { std::string name; Benchmark* benchmark; AggregationReportMode aggregation_report_mode; @@ -31,10 +32,13 @@ struct Benchmark::Instance { double min_time; size_t iterations; int threads; // Number of concurrent threads to us + + State Run(size_t iters, int thread_id, internal::ThreadTimer* timer, + internal::ThreadManager* manager) const; }; bool FindBenchmarksInternal(const std::string& re, - std::vector* benchmarks, + std::vector* benchmarks, std::ostream* Err); bool IsZero(double n); diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index ffe97a1416..a85a4b44d8 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -78,7 +78,7 @@ class BenchmarkFamilies { // Extract the list of benchmark instances that match the specified // regular expression. bool FindBenchmarks(std::string re, - std::vector* benchmarks, + std::vector* benchmarks, std::ostream* Err); private: @@ -107,7 +107,7 @@ void BenchmarkFamilies::ClearBenchmarks() { } bool BenchmarkFamilies::FindBenchmarks( - std::string spec, std::vector* benchmarks, + std::string spec, std::vector* benchmarks, std::ostream* ErrStream) { CHECK(ErrStream); auto& Err = *ErrStream; @@ -152,7 +152,7 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { - Benchmark::Instance instance; + BenchmarkInstance instance; instance.name = family->name_; instance.benchmark = family.get(); instance.aggregation_report_mode = family->aggregation_report_mode_; @@ -225,7 +225,7 @@ Benchmark* RegisterBenchmarkInternal(Benchmark* bench) { // FIXME: This function is a hack so that benchmark.cc can access // `BenchmarkFamilies` bool FindBenchmarksInternal(const std::string& re, - std::vector* benchmarks, + std::vector* benchmarks, std::ostream* Err) { return BenchmarkFamilies::GetInstance()->FindBenchmarks(re, benchmarks, Err); } From d8c0f27448dfad95b94285e612bba1f7c55c9dd0 Mon Sep 17 00:00:00 2001 From: Victor Costan Date: Mon, 1 Oct 2018 05:00:13 -0700 Subject: [PATCH 092/330] Fix possible loss of data warnings in MSVC. (#694) --- include/benchmark/benchmark.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 609dc863e3..157a71794a 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -555,13 +555,13 @@ class State { BENCHMARK_ALWAYS_INLINE void SetBytesProcessed(int64_t bytes) { counters["bytes_per_second"] = - Counter(bytes, Counter::kIsRate, Counter::kIs1024); + Counter(static_cast(bytes), Counter::kIsRate, Counter::kIs1024); } BENCHMARK_ALWAYS_INLINE int64_t bytes_processed() const { if (counters.find("bytes_per_second") != counters.end()) - return counters.at("bytes_per_second"); + return static_cast(counters.at("bytes_per_second")); return 0; } @@ -584,13 +584,14 @@ class State { // REQUIRES: a benchmark has exited its benchmarking loop. BENCHMARK_ALWAYS_INLINE void SetItemsProcessed(int64_t items) { - counters["items_per_second"] = Counter(items, benchmark::Counter::kIsRate); + counters["items_per_second"] = + Counter(static_cast(items), benchmark::Counter::kIsRate); } BENCHMARK_ALWAYS_INLINE int64_t items_processed() const { if (counters.find("items_per_second") != counters.end()) - return counters.at("items_per_second"); + return static_cast(counters.at("items_per_second")); return 0; } From a8082de5dfdd2132e8bda432314efbce3f280f99 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 1 Oct 2018 17:51:08 +0300 Subject: [PATCH 093/330] [NFC] Refactor RunBenchmark() (#690) Ok, so, i'm still trying to get to the state when it will be a trivial change to report all the separate iterations. The old code (LHS of the diff) was rather convoluted i'd say. I have tried to refactor it a bit into *small* logical chunks, with proper comments. As far as i can tell, i preserved the intent of the code, what it was doing before. The road forward still isn't clear, but i'm quite sure it's not with the old code :) --- src/benchmark.cc | 222 +--------------------- src/benchmark_api_internal.h | 1 + src/benchmark_runner.cc | 353 +++++++++++++++++++++++++++++++++++ src/benchmark_runner.h | 51 +++++ 4 files changed, 410 insertions(+), 217 deletions(-) create mode 100644 src/benchmark_runner.cc create mode 100644 src/benchmark_runner.h diff --git a/src/benchmark.cc b/src/benchmark.cc index ffa75889d5..36db34e7eb 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -14,6 +14,7 @@ #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" +#include "benchmark_runner.h" #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS @@ -113,226 +114,11 @@ DEFINE_int32(v, 0, "The level of verbose logging to output"); namespace benchmark { -namespace { -static const size_t kMaxIterations = 1000000000; - -static MemoryManager* memory_manager = nullptr; -} // end namespace - namespace internal { +// FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} -namespace { - -BenchmarkReporter::Run CreateRunReport( - const benchmark::internal::BenchmarkInstance& b, - const internal::ThreadManager::Result& results, size_t memory_iterations, - const MemoryManager::Result& memory_result, double seconds) { - // Create report about this benchmark run. - BenchmarkReporter::Run report; - - report.run_name = b.name; - report.error_occurred = results.has_error_; - report.error_message = results.error_message_; - report.report_label = results.report_label_; - // This is the total iterations across all threads. - report.iterations = results.iterations; - report.time_unit = b.time_unit; - - if (!report.error_occurred) { - if (b.use_manual_time) { - report.real_accumulated_time = results.manual_time_used; - } else { - report.real_accumulated_time = results.real_time_used; - } - report.cpu_accumulated_time = results.cpu_time_used; - report.complexity_n = results.complexity_n; - report.complexity = b.complexity; - report.complexity_lambda = b.complexity_lambda; - report.statistics = b.statistics; - report.counters = results.counters; - - if (memory_iterations > 0) { - report.has_memory_result = true; - report.allocs_per_iter = - memory_iterations ? static_cast(memory_result.num_allocs) / - memory_iterations - : 0; - report.max_bytes_used = memory_result.max_bytes_used; - } - - internal::Finish(&report.counters, results.iterations, seconds, b.threads); - } - return report; -} - -// Execute one thread of benchmark b for the specified number of iterations. -// Adds the stats collected for the thread into *total. -void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, - ThreadManager* manager) { - internal::ThreadTimer timer; - State st = b->Run(iters, thread_id, &timer, manager); - CHECK(st.iterations() >= st.max_iterations) - << "Benchmark returned before State::KeepRunning() returned false!"; - { - MutexLock l(manager->GetBenchmarkMutex()); - internal::ThreadManager::Result& results = manager->results; - results.iterations += st.iterations(); - results.cpu_time_used += timer.cpu_time_used(); - results.real_time_used += timer.real_time_used(); - results.manual_time_used += timer.manual_time_used(); - results.complexity_n += st.complexity_length_n(); - internal::Increment(&results.counters, st.counters); - } - manager->NotifyThreadComplete(); -} - -struct RunResults { - std::vector non_aggregates; - std::vector aggregates_only; - - bool display_report_aggregates_only = false; - bool file_report_aggregates_only = false; -}; - -RunResults RunBenchmark( - const benchmark::internal::BenchmarkInstance& b, - std::vector* complexity_reports) { - RunResults run_results; - - const bool has_explicit_iteration_count = b.iterations != 0; - size_t iters = has_explicit_iteration_count ? b.iterations : 1; - std::unique_ptr manager; - std::vector pool(b.threads - 1); - const int repeats = - b.repetitions != 0 ? b.repetitions : FLAGS_benchmark_repetitions; - if (repeats != 1) { - run_results.display_report_aggregates_only = - (FLAGS_benchmark_report_aggregates_only || - FLAGS_benchmark_display_aggregates_only); - run_results.file_report_aggregates_only = - FLAGS_benchmark_report_aggregates_only; - if (b.aggregation_report_mode != internal::ARM_Unspecified) { - run_results.display_report_aggregates_only = - (b.aggregation_report_mode & - internal::ARM_DisplayReportAggregatesOnly); - run_results.file_report_aggregates_only = - (b.aggregation_report_mode & internal::ARM_FileReportAggregatesOnly); - } - } - for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { - for (;;) { - // Try benchmark - VLOG(2) << "Running " << b.name << " for " << iters << "\n"; - - manager.reset(new internal::ThreadManager(b.threads)); - for (std::size_t ti = 0; ti < pool.size(); ++ti) { - pool[ti] = std::thread(&RunInThread, &b, iters, - static_cast(ti + 1), manager.get()); - } - RunInThread(&b, iters, 0, manager.get()); - manager->WaitForAllThreads(); - for (std::thread& thread : pool) thread.join(); - internal::ThreadManager::Result results; - { - MutexLock l(manager->GetBenchmarkMutex()); - results = manager->results; - } - manager.reset(); - // Adjust real/manual time stats since they were reported per thread. - results.real_time_used /= b.threads; - results.manual_time_used /= b.threads; - - VLOG(2) << "Ran in " << results.cpu_time_used << "/" - << results.real_time_used << "\n"; - - // Base decisions off of real time if requested by this benchmark. - double seconds = results.cpu_time_used; - if (b.use_manual_time) { - seconds = results.manual_time_used; - } else if (b.use_real_time) { - seconds = results.real_time_used; - } - - const double min_time = - !IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time; - - // clang-format off - // turn off clang-format since it mangles prettiness here - // Determine if this run should be reported; Either it has - // run for a sufficient amount of time or because an error was reported. - const bool should_report = repetition_num > 0 - || has_explicit_iteration_count // An exact iteration count was requested - || results.has_error_ - || iters >= kMaxIterations // No chance to try again, we hit the limit. - || seconds >= min_time // the elapsed time is large enough - // CPU time is specified but the elapsed real time greatly exceeds the - // minimum time. Note that user provided timers are except from this - // sanity check. - || ((results.real_time_used >= 5 * min_time) && !b.use_manual_time); - // clang-format on - - if (should_report) { - MemoryManager::Result memory_result; - size_t memory_iterations = 0; - if (memory_manager != nullptr) { - // Only run a few iterations to reduce the impact of one-time - // allocations in benchmarks that are not properly managed. - memory_iterations = std::min(16, iters); - memory_manager->Start(); - manager.reset(new internal::ThreadManager(1)); - RunInThread(&b, memory_iterations, 0, manager.get()); - manager->WaitForAllThreads(); - manager.reset(); - - memory_manager->Stop(&memory_result); - } - - BenchmarkReporter::Run report = CreateRunReport( - b, results, memory_iterations, memory_result, seconds); - if (!report.error_occurred && b.complexity != oNone) - complexity_reports->push_back(report); - run_results.non_aggregates.push_back(report); - break; - } - - // See how much iterations should be increased by - // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = min_time * 1.4 / std::max(seconds, 1e-9); - // If our last run was at least 10% of FLAGS_benchmark_min_time then we - // use the multiplier directly. Otherwise we use at most 10 times - // expansion. - // NOTE: When the last run was at least 10% of the min time the max - // expansion should be 14x. - bool is_significant = (seconds / min_time) > 0.1; - multiplier = is_significant ? multiplier : std::min(10.0, multiplier); - if (multiplier <= 1.0) multiplier = 2.0; - double next_iters = std::max(multiplier * iters, iters + 1.0); - if (next_iters > kMaxIterations) { - next_iters = kMaxIterations; - } - VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; - iters = static_cast(next_iters + 0.5); - } - } - - // Calculate additional statistics - run_results.aggregates_only = ComputeStats(run_results.non_aggregates); - - // Maybe calculate complexity report - if ((b.complexity != oNone) && b.last_benchmark_instance) { - auto additional_run_stats = ComputeBigO(*complexity_reports); - run_results.aggregates_only.insert(run_results.aggregates_only.end(), - additional_run_stats.begin(), - additional_run_stats.end()); - complexity_reports->clear(); - } - - return run_results; -} - -} // namespace } // namespace internal State::State(size_t max_iters, const std::vector& ranges, int thread_i, @@ -610,7 +396,9 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, return benchmarks.size(); } -void RegisterMemoryManager(MemoryManager* manager) { memory_manager = manager; } +void RegisterMemoryManager(MemoryManager* manager) { + internal::memory_manager = manager; +} namespace internal { diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index fe492c7eb6..0524a85c01 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -2,6 +2,7 @@ #define BENCHMARK_API_INTERNAL_H #include "benchmark/benchmark.h" +#include "commandlineflags.h" #include #include diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc new file mode 100644 index 0000000000..12391fcf44 --- /dev/null +++ b/src/benchmark_runner.cc @@ -0,0 +1,353 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark_runner.h" +#include "benchmark/benchmark.h" +#include "benchmark_api_internal.h" +#include "internal_macros.h" + +#ifndef BENCHMARK_OS_WINDOWS +#ifndef BENCHMARK_OS_FUCHSIA +#include +#endif +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "colorprint.h" +#include "commandlineflags.h" +#include "complexity.h" +#include "counter.h" +#include "internal_macros.h" +#include "log.h" +#include "mutex.h" +#include "re.h" +#include "statistics.h" +#include "string_util.h" +#include "thread_manager.h" +#include "thread_timer.h" + +namespace benchmark { + +namespace internal { + +MemoryManager* memory_manager = nullptr; + +namespace { + +static const size_t kMaxIterations = 1000000000; + +BenchmarkReporter::Run CreateRunReport( + const benchmark::internal::BenchmarkInstance& b, + const internal::ThreadManager::Result& results, size_t memory_iterations, + const MemoryManager::Result& memory_result, double seconds) { + // Create report about this benchmark run. + BenchmarkReporter::Run report; + + report.run_name = b.name; + report.error_occurred = results.has_error_; + report.error_message = results.error_message_; + report.report_label = results.report_label_; + // This is the total iterations across all threads. + report.iterations = results.iterations; + report.time_unit = b.time_unit; + + if (!report.error_occurred) { + if (b.use_manual_time) { + report.real_accumulated_time = results.manual_time_used; + } else { + report.real_accumulated_time = results.real_time_used; + } + report.cpu_accumulated_time = results.cpu_time_used; + report.complexity_n = results.complexity_n; + report.complexity = b.complexity; + report.complexity_lambda = b.complexity_lambda; + report.statistics = b.statistics; + report.counters = results.counters; + + if (memory_iterations > 0) { + report.has_memory_result = true; + report.allocs_per_iter = + memory_iterations ? static_cast(memory_result.num_allocs) / + memory_iterations + : 0; + report.max_bytes_used = memory_result.max_bytes_used; + } + + internal::Finish(&report.counters, results.iterations, seconds, b.threads); + } + return report; +} + +// Execute one thread of benchmark b for the specified number of iterations. +// Adds the stats collected for the thread into *total. +void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, + ThreadManager* manager) { + internal::ThreadTimer timer; + State st = b->Run(iters, thread_id, &timer, manager); + CHECK(st.iterations() >= st.max_iterations) + << "Benchmark returned before State::KeepRunning() returned false!"; + { + MutexLock l(manager->GetBenchmarkMutex()); + internal::ThreadManager::Result& results = manager->results; + results.iterations += st.iterations(); + results.cpu_time_used += timer.cpu_time_used(); + results.real_time_used += timer.real_time_used(); + results.manual_time_used += timer.manual_time_used(); + results.complexity_n += st.complexity_length_n(); + internal::Increment(&results.counters, st.counters); + } + manager->NotifyThreadComplete(); +} + +class BenchmarkRunner { + public: + BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, + std::vector* complexity_reports_) + : b(b_), + complexity_reports(*complexity_reports_), + min_time(!IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time), + repeats(b.repetitions != 0 ? b.repetitions + : FLAGS_benchmark_repetitions), + has_explicit_iteration_count(b.iterations != 0), + pool(b.threads - 1), + iters(has_explicit_iteration_count ? b.iterations : 1) { + if (repeats != 1) { + run_results.display_report_aggregates_only = + (FLAGS_benchmark_report_aggregates_only || + FLAGS_benchmark_display_aggregates_only); + run_results.file_report_aggregates_only = + FLAGS_benchmark_report_aggregates_only; + if (b.aggregation_report_mode != internal::ARM_Unspecified) { + run_results.display_report_aggregates_only = + (b.aggregation_report_mode & + internal::ARM_DisplayReportAggregatesOnly); + run_results.file_report_aggregates_only = + (b.aggregation_report_mode & + internal::ARM_FileReportAggregatesOnly); + } + } + + for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { + const bool is_the_first_repetition = repetition_num == 0; + DoOneRepetition(is_the_first_repetition); + } + + // Calculate additional statistics + run_results.aggregates_only = ComputeStats(run_results.non_aggregates); + + // Maybe calculate complexity report + if ((b.complexity != oNone) && b.last_benchmark_instance) { + auto additional_run_stats = ComputeBigO(complexity_reports); + run_results.aggregates_only.insert(run_results.aggregates_only.end(), + additional_run_stats.begin(), + additional_run_stats.end()); + complexity_reports.clear(); + } + } + + RunResults&& get_results() { return std::move(run_results); } + + private: + RunResults run_results; + + const benchmark::internal::BenchmarkInstance& b; + std::vector& complexity_reports; + + const double min_time; + const int repeats; + const bool has_explicit_iteration_count; + + std::vector pool; + + size_t iters; // preserved between repetitions! + // So only the first repetition has to find/calculate it, + // the other repetitions will just use that precomputed iteration count. + + struct IterationResults { + internal::ThreadManager::Result results; + size_t iters; + double seconds; + }; + IterationResults DoNIterations() { + VLOG(2) << "Running " << b.name << " for " << iters << "\n"; + + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(b.threads)); + + // Run all but one thread in separate threads + for (std::size_t ti = 0; ti < pool.size(); ++ti) { + pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), + manager.get()); + } + // And run one thread here directly. + // (If we were asked to run just one thread, we don't create new threads.) + // Yes, we need to do this here *after* we start the separate threads. + RunInThread(&b, iters, 0, manager.get()); + + // The main thread has finished. Now let's wait for the other threads. + manager->WaitForAllThreads(); + for (std::thread& thread : pool) thread.join(); + + IterationResults i; + // Acquire the measurements/counters from the manager, UNDER THE LOCK! + { + MutexLock l(manager->GetBenchmarkMutex()); + i.results = manager->results; + } + + // And get rid of the manager. + manager.reset(); + + // Adjust real/manual time stats since they were reported per thread. + i.results.real_time_used /= b.threads; + i.results.manual_time_used /= b.threads; + + VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" + << i.results.real_time_used << "\n"; + + // So for how long were we running? + i.iters = iters; + // Base decisions off of real time if requested by this benchmark. + i.seconds = i.results.cpu_time_used; + if (b.use_manual_time) { + i.seconds = i.results.manual_time_used; + } else if (b.use_real_time) { + i.seconds = i.results.real_time_used; + } + + return i; + } + + size_t PredictNumItersNeeded(const IterationResults& i) const { + // See how much iterations should be increased by. + // Note: Avoid division by zero with max(seconds, 1ns). + double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); + // If our last run was at least 10% of FLAGS_benchmark_min_time then we + // use the multiplier directly. + // Otherwise we use at most 10 times expansion. + // NOTE: When the last run was at least 10% of the min time the max + // expansion should be 14x. + bool is_significant = (i.seconds / min_time) > 0.1; + multiplier = is_significant ? multiplier : std::min(10.0, multiplier); + if (multiplier <= 1.0) multiplier = 2.0; + + // So what seems to be the sufficiently-large iteration count? Round up. + const size_t max_next_iters = + 0.5 + std::max(multiplier * i.iters, i.iters + 1.0); + // But we do have *some* sanity limits though.. + const size_t next_iters = std::min(max_next_iters, kMaxIterations); + + VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; + return next_iters; // round up before conversion to integer. + } + + bool ShouldReportIterationResults(const IterationResults& i) const { + // Determine if this run should be reported; + // Either it has run for a sufficient amount of time + // or because an error was reported. + return i.results.has_error_ || + i.iters >= kMaxIterations || // Too many iterations already. + i.seconds >= min_time || // The elapsed time is large enough. + // CPU time is specified but the elapsed real time greatly exceeds + // the minimum time. + // Note that user provided timers are except from this sanity check. + ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time); + } + + void DoOneRepetition(bool is_the_first_repetition) { + IterationResults i; + + // We *may* be gradually increasing the length (iteration count) + // of the benchmark until we decide the results are significant. + // And once we do, we report those last results and exit. + // Please do note that the if there are repetitions, the iteration count + // is *only* calculated for the *first* repetition, and other repetitions + // simply use that precomputed iteration count. + for (;;) { + i = DoNIterations(); + + // Do we consider the results to be significant? + // If we are doing repetitions, and the first repetition was already done, + // it has calculated the correct iteration time, so we have run that very + // iteration count just now. No need to calculate anything. Just report. + // Else, the normal rules apply. + const bool results_are_significant = !is_the_first_repetition || + has_explicit_iteration_count || + ShouldReportIterationResults(i); + + if (results_are_significant) break; // Good, let's report them! + + // Nope, bad iteration. Let's re-estimate the hopefully-sufficient + // iteration count, and run the benchmark again... + + iters = PredictNumItersNeeded(i); + assert(iters > i.iters && + "if we did more iterations than we want to do the next time, " + "then we should have accepted the current iteration run."); + } + + // Oh, one last thing, we need to also produce the 'memory measurements'.. + MemoryManager::Result memory_result; + size_t memory_iterations = 0; + if (memory_manager != nullptr) { + // Only run a few iterations to reduce the impact of one-time + // allocations in benchmarks that are not properly managed. + memory_iterations = std::min(16, iters); + memory_manager->Start(); + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(1)); + RunInThread(&b, memory_iterations, 0, manager.get()); + manager->WaitForAllThreads(); + manager.reset(); + + memory_manager->Stop(&memory_result); + } + + // Ok, now actualy report. + BenchmarkReporter::Run report = CreateRunReport( + b, i.results, memory_iterations, memory_result, i.seconds); + + if (!report.error_occurred && b.complexity != oNone) + complexity_reports.push_back(report); + + run_results.non_aggregates.push_back(report); + } +}; + +} // end namespace + +RunResults RunBenchmark( + const benchmark::internal::BenchmarkInstance& b, + std::vector* complexity_reports) { + internal::BenchmarkRunner r(b, complexity_reports); + return r.get_results(); +} + +} // end namespace internal + +} // end namespace benchmark diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h new file mode 100644 index 0000000000..96e8282a11 --- /dev/null +++ b/src/benchmark_runner.h @@ -0,0 +1,51 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_RUNNER_H_ +#define BENCHMARK_RUNNER_H_ + +#include "benchmark_api_internal.h" +#include "internal_macros.h" + +DECLARE_double(benchmark_min_time); + +DECLARE_int32(benchmark_repetitions); + +DECLARE_bool(benchmark_report_aggregates_only); + +DECLARE_bool(benchmark_display_aggregates_only); + +namespace benchmark { + +namespace internal { + +extern MemoryManager* memory_manager; + +struct RunResults { + std::vector non_aggregates; + std::vector aggregates_only; + + bool display_report_aggregates_only = false; + bool file_report_aggregates_only = false; +}; + +RunResults RunBenchmark( + const benchmark::internal::BenchmarkInstance& b, + std::vector* complexity_reports); + +} // namespace internal + +} // end namespace benchmark + +#endif // BENCHMARK_RUNNER_H_ From 9ffb8df6c5d62a677ee27d19cee93a6f3197fd19 Mon Sep 17 00:00:00 2001 From: Gregorio Litenstein Date: Fri, 5 Oct 2018 12:44:02 -0300 Subject: [PATCH 094/330] Fix Clang Detection (#697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For several versions now, CMake by default refers to macOS’ Clang as AppleClang instead of just Clang, which would fail STREQUAL. Fixed by changing it to MATCHES. --- CMakeLists.txt | 4 ++-- cmake/HandleGTest.cmake | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fff85b14b1..1c57db9d89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,7 +193,7 @@ else() if (GCC_RANLIB) set(CMAKE_RANLIB ${GCC_RANLIB}) endif() - elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") + elseif("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") include(llvm-toolchain) endif() endif() @@ -218,7 +218,7 @@ else() endif() if (BENCHMARK_USE_LIBCXX) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") add_cxx_compiler_flag(-stdlib=libc++) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 7ce1a633d6..8c03b751da 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -5,7 +5,7 @@ macro(build_external_gtest) include(ExternalProject) set(GTEST_FLAGS "") if (BENCHMARK_USE_LIBCXX) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") list(APPEND GTEST_FLAGS -stdlib=libc++) else() message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++") From 8503dfe537a1fc60a88116ee024f57e9fb5131a1 Mon Sep 17 00:00:00 2001 From: "Ilya A. Kriveshko" Date: Mon, 8 Oct 2018 04:33:21 -0400 Subject: [PATCH 095/330] benchmark_color: fix auto option (#559) (#699) As prevously written, "--benchmark_color=auto" was treated as true, because IsTruthyFlagValue("auto") returned true. The fix is to rely on IsColorTerminal test only if the flag value is "auto", and fall back to IsTruthyFlagValue otherwise. I also integrated force_no_color check into the same block. --- src/benchmark.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 36db34e7eb..67b6c7a792 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -312,15 +312,20 @@ bool IsZero(double n) { ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { int output_opts = ConsoleReporter::OO_Defaults; - if ((FLAGS_benchmark_color == "auto" && IsColorTerminal()) || - IsTruthyFlagValue(FLAGS_benchmark_color)) { + auto is_benchmark_color = [force_no_color] () -> bool { + if (force_no_color) { + return false; + } + if (FLAGS_benchmark_color == "auto") { + return IsColorTerminal(); + } + return IsTruthyFlagValue(FLAGS_benchmark_color); + }; + if (is_benchmark_color()) { output_opts |= ConsoleReporter::OO_Color; } else { output_opts &= ~ConsoleReporter::OO_Color; } - if (force_no_color) { - output_opts &= ~ConsoleReporter::OO_Color; - } if (FLAGS_benchmark_counters_tabular) { output_opts |= ConsoleReporter::OO_Tabular; } else { From 609752306fca311ee1caca2306696dfe245bdce4 Mon Sep 17 00:00:00 2001 From: Olzhas Kaiyrakhmet Date: Sat, 13 Oct 2018 16:51:51 +0900 Subject: [PATCH 096/330] Fix SOURCE_DIR in HandleGTest.cmake (#703) * Fix SOURCE_DIR in HandleGTest.cmake If benchmark added as cmake subproject, HandleGTest throws an error as does return absolute source dir. Change it to , so it will be refering to it's own source dir. --- cmake/HandleGTest.cmake | 43 ++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 8c03b751da..aad4d7ecbc 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -75,27 +75,34 @@ macro(build_external_gtest) endforeach() endmacro(build_external_gtest) +macro(build_gtest_from_dir) + set(INSTALL_GTEST OFF CACHE INTERNAL "") + set(INSTALL_GMOCK OFF CACHE INTERNAL "") + add_subdirectory(${GTEST_ROOT} build) + set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) + foreach(HEADER test mock) + # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we + # have to add the paths ourselves. + set(HFILE g${HEADER}/g${HEADER}.h) + set(HPATH ${GTEST_ROOT}/google${HEADER}/include) + find_path(HEADER_PATH_${HEADER} ${HFILE} + NO_DEFAULT_PATHS + HINTS ${HPATH} + ) + if (NOT HEADER_PATH_${HEADER}) + message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") + endif() + list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) + endforeach() +endmacro() + if (BENCHMARK_ENABLE_GTEST_TESTS) if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest) set(GTEST_ROOT "${CMAKE_SOURCE_DIR}/googletest") - set(INSTALL_GTEST OFF CACHE INTERNAL "") - set(INSTALL_GMOCK OFF CACHE INTERNAL "") - add_subdirectory(${CMAKE_SOURCE_DIR}/googletest) - set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) - foreach(HEADER test mock) - # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we - # have to add the paths ourselves. - set(HFILE g${HEADER}/g${HEADER}.h) - set(HPATH ${GTEST_ROOT}/google${HEADER}/include) - find_path(HEADER_PATH_${HEADER} ${HFILE} - NO_DEFAULT_PATHS - HINTS ${HPATH} - ) - if (NOT HEADER_PATH_${HEADER}) - message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") - endif() - list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) - endforeach() + build_gtest_from_dir() + elseif (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest) + set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest") + build_gtest_from_dir() elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) build_external_gtest() else() From 8356d646bf953d6715e04b7b3714081cb57e8f13 Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 13 Oct 2018 00:53:25 -0700 Subject: [PATCH 097/330] Revert "Fix SOURCE_DIR in HandleGTest.cmake (#703)" (#704) This reverts commit 609752306fca311ee1caca2306696dfe245bdce4. --- cmake/HandleGTest.cmake | 43 +++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index aad4d7ecbc..8c03b751da 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -75,34 +75,27 @@ macro(build_external_gtest) endforeach() endmacro(build_external_gtest) -macro(build_gtest_from_dir) - set(INSTALL_GTEST OFF CACHE INTERNAL "") - set(INSTALL_GMOCK OFF CACHE INTERNAL "") - add_subdirectory(${GTEST_ROOT} build) - set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) - foreach(HEADER test mock) - # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we - # have to add the paths ourselves. - set(HFILE g${HEADER}/g${HEADER}.h) - set(HPATH ${GTEST_ROOT}/google${HEADER}/include) - find_path(HEADER_PATH_${HEADER} ${HFILE} - NO_DEFAULT_PATHS - HINTS ${HPATH} - ) - if (NOT HEADER_PATH_${HEADER}) - message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") - endif() - list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) - endforeach() -endmacro() - if (BENCHMARK_ENABLE_GTEST_TESTS) if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest) set(GTEST_ROOT "${CMAKE_SOURCE_DIR}/googletest") - build_gtest_from_dir() - elseif (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest) - set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest") - build_gtest_from_dir() + set(INSTALL_GTEST OFF CACHE INTERNAL "") + set(INSTALL_GMOCK OFF CACHE INTERNAL "") + add_subdirectory(${CMAKE_SOURCE_DIR}/googletest) + set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) + foreach(HEADER test mock) + # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we + # have to add the paths ourselves. + set(HFILE g${HEADER}/g${HEADER}.h) + set(HPATH ${GTEST_ROOT}/google${HEADER}/include) + find_path(HEADER_PATH_${HEADER} ${HFILE} + NO_DEFAULT_PATHS + HINTS ${HPATH} + ) + if (NOT HEADER_PATH_${HEADER}) + message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") + endif() + list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) + endforeach() elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) build_external_gtest() else() From d731697a5da9cec25034683ed9807304595ea72c Mon Sep 17 00:00:00 2001 From: Olzhas Kaiyrakhmet Date: Sat, 13 Oct 2018 17:06:41 +0900 Subject: [PATCH 098/330] Fix SOURCE_DIR in HandleGTest.cmake (#705) If benchmark added as cmake subproject, HandleGTest throws an error as does return absolute source dir. Change it to , so it will be refering to it's own source dir. Also see PR #703. --- cmake/HandleGTest.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake index 8c03b751da..b9c14436db 100644 --- a/cmake/HandleGTest.cmake +++ b/cmake/HandleGTest.cmake @@ -76,11 +76,11 @@ macro(build_external_gtest) endmacro(build_external_gtest) if (BENCHMARK_ENABLE_GTEST_TESTS) - if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest) - set(GTEST_ROOT "${CMAKE_SOURCE_DIR}/googletest") + if (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest) + set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest") set(INSTALL_GTEST OFF CACHE INTERNAL "") set(INSTALL_GMOCK OFF CACHE INTERNAL "") - add_subdirectory(${CMAKE_SOURCE_DIR}/googletest) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/googletest) set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) foreach(HEADER test mock) # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we From 9cacec8e78fb359f7148b7848cd0b305bdc80732 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 18 Oct 2018 15:03:17 +0300 Subject: [PATCH 099/330] [NFC] RunBenchmarks(): s/has_repetitions/might_have_aggregates/ (#707) That is the real purpose of that bool. A follow-up change will make it consider something else other than repetitions. --- src/benchmark.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 67b6c7a792..a22967b72d 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -228,18 +228,18 @@ void RunBenchmarks(const std::vector& benchmarks, CHECK(display_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. - bool has_repetitions = FLAGS_benchmark_repetitions > 1; + bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; size_t name_field_width = 10; size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { name_field_width = std::max(name_field_width, benchmark.name.size()); - has_repetitions |= benchmark.repetitions > 1; + might_have_aggregates |= benchmark.repetitions > 1; for (const auto& Stat : *benchmark.statistics) stat_field_width = std::max(stat_field_width, Stat.name_.size()); } - if (has_repetitions) name_field_width += 1 + stat_field_width; + if (might_have_aggregates) name_field_width += 1 + stat_field_width; // Print header here BenchmarkReporter::Context context; From 99d1356c041f49dccfc55104e79e282f815f6849 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 18 Oct 2018 15:08:59 +0300 Subject: [PATCH 100/330] [NFC] BenchmarkRunner: always populate *_report_aggregates_only bools. (#708) It is better to let the RunBenchmarks(), report() decide whether to actually *only* output aggregates or not, depending on whether there are actually aggregates. It's subtle indeed. Previously, `BenchmarkRunner()` always said that "if there are no repetitions, then you should never output only the repetitions". And the `report()` simply assumed that the `report_aggregates_only` bool it received makes sense, and simply used it. Now, the logic is the same, but the blame has shifted. `BenchmarkRunner()` always propagates what those benchmarks would have wanted to happen wrt the aggregates. And the `report()` lambda has to actually consider both the `report_aggregates_only` bool, and it's meaningfulness. To put it in the context of the patch series - if the repetition count was `1`, but `*_report_aggregates_only` was set to `true`, and we capture each iteration separately, then we will compute the aggregates, but then output everything, both the iteration, and aggregates, despite `*_report_aggregates_only` being set to `true`. --- src/benchmark.cc | 4 ++-- src/benchmark_runner.cc | 21 +++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index a22967b72d..410493fded 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -267,8 +267,8 @@ void RunBenchmarks(const std::vector& benchmarks, auto report = [&run_results](BenchmarkReporter* reporter, bool report_aggregates_only) { assert(reporter); - assert( - !(report_aggregates_only && run_results.aggregates_only.empty())); + // If there are no aggregates, do output non-aggregates. + report_aggregates_only &= !run_results.aggregates_only.empty(); if (!report_aggregates_only) reporter->ReportRuns(run_results.non_aggregates); if (!run_results.aggregates_only.empty()) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 12391fcf44..38faeec8e3 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -136,20 +136,17 @@ class BenchmarkRunner { has_explicit_iteration_count(b.iterations != 0), pool(b.threads - 1), iters(has_explicit_iteration_count ? b.iterations : 1) { - if (repeats != 1) { + run_results.display_report_aggregates_only = + (FLAGS_benchmark_report_aggregates_only || + FLAGS_benchmark_display_aggregates_only); + run_results.file_report_aggregates_only = + FLAGS_benchmark_report_aggregates_only; + if (b.aggregation_report_mode != internal::ARM_Unspecified) { run_results.display_report_aggregates_only = - (FLAGS_benchmark_report_aggregates_only || - FLAGS_benchmark_display_aggregates_only); + (b.aggregation_report_mode & + internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = - FLAGS_benchmark_report_aggregates_only; - if (b.aggregation_report_mode != internal::ARM_Unspecified) { - run_results.display_report_aggregates_only = - (b.aggregation_report_mode & - internal::ARM_DisplayReportAggregatesOnly); - run_results.file_report_aggregates_only = - (b.aggregation_report_mode & - internal::ARM_FileReportAggregatesOnly); - } + (b.aggregation_report_mode & internal::ARM_FileReportAggregatesOnly); } for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { From 507c06e636e0d21ca900024df464e965d3e1b669 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 18 Oct 2018 17:17:14 +0300 Subject: [PATCH 101/330] Aggregates: use non-aggregate count as iteration count. (#706) It is incorrect to say that an aggregate is computed over run's iterations, because those iterations already got averaged. Similarly, if there are N repetitions with 1 iterations each, an aggregate will be computed over N measurements, not 1. Thus it is best to simply use the count of separate reports. Fixes #586. --- src/statistics.cc | 20 ++- test/output_test_helper.cc | 8 +- test/reporter_output_test.cc | 217 +++++++++++++++++---------- test/user_counters_thousands_test.cc | 12 +- 4 files changed, 170 insertions(+), 87 deletions(-) diff --git a/src/statistics.cc b/src/statistics.cc index 8c90a44f01..e821aec18b 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -141,6 +141,9 @@ std::vector ComputeStats( } } + const double iteration_rescale_factor = + double(reports.size()) / double(run_iterations); + for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; @@ -148,15 +151,30 @@ std::vector ComputeStats( data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.aggregate_name = Stat.name_; data.report_label = report_label; - data.iterations = run_iterations; + + // It is incorrect to say that an aggregate is computed over + // run's iterations, because those iterations already got averaged. + // Similarly, if there are N repetitions with 1 iterations each, + // an aggregate will be computed over N measurements, not 1. + // Thus it is best to simply use the count of separate reports. + data.iterations = reports.size(); data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat); data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat); + // We will divide these times by data.iterations when reporting, but the + // data.iterations is not nessesairly the scale of these measurements, + // because in each repetition, these timers are sum over all the iterations. + // And if we want to say that the stats are over N repetitions and not + // M iterations, we need to multiply these by (N/M). + data.real_accumulated_time *= iteration_rescale_factor; + data.cpu_accumulated_time *= iteration_rescale_factor; + data.time_unit = reports[0].time_unit; // user counters for (auto const& kv : counter_stats) { + // Do *NOT* rescale the custom counters. They are already properly scaled. const auto uc_stat = Stat.compute_(kv.second.s); auto c = Counter(uc_stat, counter_stats[kv.first].c.flags, counter_stats[kv.first].c.oneK); diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index f84bd34521..f2da752e1a 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -44,9 +44,11 @@ SubMap& GetSubstitutions() { {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, - {"%time", "[ ]*[0-9]{1,6} ns"}, - {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"}, - {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, + {"%time", "[ ]*[0-9]+ ns"}, + {"%console_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns [ ]*[0-9]+"}, + {"%console_time_only_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns"}, + {"%console_us_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us [ ]*[0-9]+"}, + {"%console_us_time_only_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us"}, {"%csv_header", "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "items_per_second,label,error_occurred,error_message"}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 50c1f758ae..80314b3431 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -239,11 +239,12 @@ void BM_Repeat(benchmark::State& state) { } // need two repetitions min to be able to output any aggregate output BENCHMARK(BM_Repeat)->Repetitions(2); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"}, - {"^BM_Repeat/repeats:2 %console_report$"}, - {"^BM_Repeat/repeats:2_mean %console_report$"}, - {"^BM_Repeat/repeats:2_median %console_report$"}, - {"^BM_Repeat/repeats:2_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:2 %console_report$"}, + {"^BM_Repeat/repeats:2 %console_report$"}, + {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"}, + {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, + {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -254,14 +255,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}}); + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, @@ -269,12 +273,13 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); // but for two repetitions, mean and median is the same, so let's repeat.. BENCHMARK(BM_Repeat)->Repetitions(3); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3_mean %console_report$"}, - {"^BM_Repeat/repeats:3_median %console_report$"}, - {"^BM_Repeat/repeats:3_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"}, + {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, + {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -288,14 +293,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}}); + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -304,13 +312,14 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); // median differs between even/odd number of repetitions, so just to be sure BENCHMARK(BM_Repeat)->Repetitions(4); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4_mean %console_report$"}, - {"^BM_Repeat/repeats:4_median %console_report$"}, - {"^BM_Repeat/repeats:4_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"}, + {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, + {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -327,14 +336,17 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}}); + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -362,25 +374,29 @@ void BM_SummaryRepeat(benchmark::State& state) { } } BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); -ADD_CASES(TC_ConsoleOut, - {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, - {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, - {"^BM_SummaryRepeat/repeats:3_median %console_report$"}, - {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); +ADD_CASES( + TC_ConsoleOut, + {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"}, + {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"}, + {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}}); + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, @@ -394,25 +410,29 @@ void BM_SummaryDisplay(benchmark::State& state) { } } BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); -ADD_CASES(TC_ConsoleOut, - {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, - {"^BM_SummaryDisplay/repeats:2_mean %console_report$"}, - {"^BM_SummaryDisplay/repeats:2_median %console_report$"}, - {"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}}); +ADD_CASES( + TC_ConsoleOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"}, + {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"}, + {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}}); + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, @@ -428,27 +448,33 @@ BENCHMARK(BM_RepeatTimeUnit) ->Repetitions(3) ->ReportAggregatesOnly() ->Unit(benchmark::kMicrosecond); -ADD_CASES(TC_ConsoleOut, - {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, - {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, - {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"}, - {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); +ADD_CASES( + TC_ConsoleOut, + {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"}, + {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ " + "]*3$"}, + {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ " + "]*3$"}}); ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}}); ADD_CASES(TC_CSVOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, @@ -465,55 +491,92 @@ const auto UserStatistics = [](const std::vector& v) { }; void BM_UserStats(benchmark::State& state) { for (auto _ : state) { + state.SetIterationTime(150 / 10e8); } } // clang-format off BENCHMARK(BM_UserStats) ->Repetitions(3) + ->Iterations(5) + ->UseManualTime() ->ComputeStatistics("", UserStatistics); // clang-format on // check that user-provided stats is calculated, and is after the default-ones // empty string as name is intentional, it would sort before anything else -ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3_mean %console_report$"}, - {"^BM_UserStats/repeats:3_median %console_report$"}, - {"^BM_UserStats/repeats:3_stddev %console_report$"}, - {"^BM_UserStats/repeats:3_ %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"mean\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3_median\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"median\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"stddev\",$", MR_Next}, - {"\"name\": \"BM_UserStats/repeats:3_\",$"}, - {"\"run_name\": \"BM_UserStats/repeats:3\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"aggregate_name\": \"\",$", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_median\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_\",%csv_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_mean [ ]* 150 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_median [ ]* 150 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_stddev [ ]* 0 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time_ " + "[ ]* 150 ns %time [ ]*3$"}}); +ADD_CASES( + TC_JSONOut, + {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/" + "manual_time_median\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/" + "manual_time_stddev\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 2c64dc4207..fa0ef97204 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -45,9 +45,9 @@ ADD_CASES( "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " "t4_1048576Base1024=1024k$"}, - {"^BM_Counters_Thousands/repeats:2_stddev %console_report " - "t0_1000000DefaultBase=0 t1_1000000Base1000=0 t2_1000000Base1024=0 " - "t3_1048576Base1000=0 t4_1048576Base1024=0$"}, + {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " + "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " + "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, }); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, @@ -82,7 +82,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next}, @@ -97,7 +97,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next}, @@ -112,7 +112,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, - {"\"iterations\": %int,$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, {"\"time_unit\": \"ns\",$", MR_Next}, From c6193afe7eb1eb7802e34833e55e1528cb65c533 Mon Sep 17 00:00:00 2001 From: Anton Gladky Date: Sun, 21 Oct 2018 10:01:42 +0200 Subject: [PATCH 102/330] Fix parsing of cpuinfo for s390 platform. (#712) s390 has another line structure for processor-field. It should be differently parsed. --- src/sysinfo.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index d4e5336fdb..d740047eea 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -404,7 +404,13 @@ int GetNumCPUs() { if (ln.empty()) continue; size_t SplitIdx = ln.find(':'); std::string value; +#if defined(__s390__) + // s390 has another format in /proc/cpuinfo + // it needs to be parsed differently + if (SplitIdx != std::string::npos) value = ln.substr(Key.size()+1,SplitIdx-Key.size()-1); +#else if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); +#endif if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { NumCPUs++; if (!value.empty()) { From a9b31c51b1ee7ec7b31438c647123c2cbac5d956 Mon Sep 17 00:00:00 2001 From: Kirill Bobyrev Date: Fri, 2 Nov 2018 19:03:49 +0300 Subject: [PATCH 103/330] Disable exceptions in Microsoft STL (#715) This is the copy of patch proposed to LLVM's copy of benchmark via https://reviews.llvm.org/D52998. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c57db9d89..771d282210 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,7 @@ if (MSVC) if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-EHs-) add_cxx_compiler_flag(-EHa-) + add_definitions(-D_HAS_EXCEPTIONS=0) endif() # Link time optimisation if (BENCHMARK_ENABLE_LTO) @@ -167,7 +168,7 @@ else() endif() # ICC17u2: overloaded virtual function "benchmark::Fixture::SetUp" is only partially overridden # (because of deprecated overload) - add_cxx_compiler_flag(-wd654) + add_cxx_compiler_flag(-wd654) add_cxx_compiler_flag(-Wthread-safety) if (HAVE_CXX_FLAG_WTHREAD_SAFETY) cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) From bb15a4e3bf4c5941ee7124d284ed9ef96e9a1c68 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 13 Nov 2018 09:56:22 +0000 Subject: [PATCH 104/330] Ensure all informational cmake messages have STATUS type --- CMakeLists.txt | 4 ++-- cmake/CXXFeatureCheck.cmake | 10 +++++----- cmake/GetGitVersion.cmake | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 771d282210..310c7ee9f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ get_git_version(GIT_VERSION) # Tell the user what versions we are using string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" VERSION ${GIT_VERSION}) -message("-- Version: ${VERSION}") +message(STATUS "Version: ${VERSION}") # The version of the libraries set(GENERIC_LIB_VERSION ${VERSION}) @@ -224,7 +224,7 @@ if (BENCHMARK_USE_LIBCXX) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") add_cxx_compiler_flag(-nostdinc++) - message("libc++ header path must be manually specified using CMAKE_CXX_FLAGS") + message(WARNING "libc++ header path must be manually specified using CMAKE_CXX_FLAGS") # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break # configuration checks such as 'find_package(Threads)' list(APPEND BENCHMARK_CXX_LINKER_FLAGS -nodefaultlibs) diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index c4c4d660f1..99b56dd623 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -28,7 +28,7 @@ function(cxx_feature_check FILE) endif() if (NOT DEFINED COMPILE_${FEATURE}) - message("-- Performing Test ${FEATURE}") + message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) try_compile(COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp @@ -42,7 +42,7 @@ function(cxx_feature_check FILE) set(RUN_${FEATURE} 1) endif() else() - message("-- Performing Test ${FEATURE}") + message(STATUS "Performing Test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} @@ -51,14 +51,14 @@ function(cxx_feature_check FILE) endif() if(RUN_${FEATURE} EQUAL 0) - message("-- Performing Test ${FEATURE} -- success") + message(STATUS "Performing Test ${FEATURE} -- success") set(HAVE_${VAR} 1 PARENT_SCOPE) add_definitions(-DHAVE_${VAR}) else() if(NOT COMPILE_${FEATURE}) - message("-- Performing Test ${FEATURE} -- failed to compile") + message(STATUS "Performing Test ${FEATURE} -- failed to compile") else() - message("-- Performing Test ${FEATURE} -- compiled but failed to run") + message(STATUS "Performing Test ${FEATURE} -- compiled but failed to run") endif() endif() endfunction() diff --git a/cmake/GetGitVersion.cmake b/cmake/GetGitVersion.cmake index 88cebe3a1c..4f10f226d7 100644 --- a/cmake/GetGitVersion.cmake +++ b/cmake/GetGitVersion.cmake @@ -49,6 +49,6 @@ function(get_git_version var) set(GIT_VERSION "v0.0.0") endif() - message("-- git Version: ${GIT_VERSION}") + message(STATUS "git Version: ${GIT_VERSION}") set(${var} ${GIT_VERSION} PARENT_SCOPE) endfunction() From 56f5cd6a729280ba7639574bd49a2bf4be7f1c4b Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 22 Nov 2018 06:38:25 +0400 Subject: [PATCH 105/330] Fix C++17 mode compilation with Apple clang (#721) --- test/output_test_helper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index f2da752e1a..6c9385ac0b 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -207,7 +207,7 @@ void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { void ResultsChecker::CheckResults(std::stringstream& output) { // first reset the stream to the start { - auto start = std::ios::streampos(0); + auto start = std::stringstream::pos_type(0); // clear before calling tellg() output.clear(); // seek to zero only when needed From c9311a44e1280853632fe2472345dd04514a2f74 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 26 Nov 2018 15:39:36 +0300 Subject: [PATCH 106/330] README.md: BM_Sequential(): the return type is 'void' Used that example as a snippet, and it took a moment to notice what needed to be changed to make it compile.. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38203958f1..360a18dc43 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the absence of multiprogramming. ```c++ -template int BM_Sequential(benchmark::State& state) { +template void BM_Sequential(benchmark::State& state) { Q q; typename Q::value_type v; for (auto _ : state) { From c9f2693ea97e94c8afcefb57d3074c6a6236ca23 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 27 Nov 2018 03:55:05 +0300 Subject: [PATCH 107/330] StrFormat() is a printf-like function, mark it as such, fix fallout. (#727) Fixes #714. --- src/benchmark_register.cc | 11 ++++++++--- src/string_util.h | 6 +++++- test/reporter_output_test.cc | 12 ++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index a85a4b44d8..f17f5b223c 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -182,14 +182,19 @@ bool BenchmarkFamilies::FindBenchmarks( } } - instance.name += StrFormat("%d", arg); + // we know that the args are always non-negative (see 'AddRange()'), + // thus print as 'unsigned'. BUT, do a cast due to the 32-bit builds. + instance.name += StrFormat("%lu", static_cast(arg)); ++arg_i; } if (!IsZero(family->min_time_)) instance.name += StrFormat("/min_time:%0.3f", family->min_time_); - if (family->iterations_ != 0) - instance.name += StrFormat("/iterations:%d", family->iterations_); + if (family->iterations_ != 0) { + instance.name += + StrFormat("/iterations:%lu", + static_cast(family->iterations_)); + } if (family->repetitions_ != 0) instance.name += StrFormat("/repeats:%d", family->repetitions_); diff --git a/src/string_util.h b/src/string_util.h index 4a5501273c..fc5f8b0304 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -12,7 +12,11 @@ void AppendHumanReadable(int n, std::string* str); std::string HumanReadableNumber(double n, double one_k = 1024.0); -std::string StrFormat(const char* format, ...); +#ifdef __GNUC__ +__attribute__((format(printf, 1, 2))) +#endif +std::string +StrFormat(const char* format, ...); inline std::ostream& StrCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { return out; diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8a45471a43..35bb9b26e9 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -219,6 +219,18 @@ ADD_CASES(TC_JSONOut, {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); +// ========================================================================= // +// ------------------------ Testing Big Args Output ------------------------ // +// ========================================================================= // + +void BM_BigArgs(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U); +ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, + {"^BM_BigArgs/2147483648 %console_report$"}}); + // ========================================================================= // // ----------------------- Testing Complexity Output ----------------------- // // ========================================================================= // From 19f7d5c2bcc98d34f4dadad04cc3c979589fc3ae Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 29 Nov 2018 01:23:25 +0300 Subject: [PATCH 108/330] README.md: complexity lambda takes int64_t arg. Fixes #719 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 360a18dc43..e4d5bf133c 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ that might be used to customize high-order term calculation. ```c++ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) - ->Range(1<<10, 1<<18)->Complexity([](int n)->double{return n; }); + ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; }); ``` ### Templated benchmarks From eafa34a5e80c352b078307be312d3fafd0a5d13e Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 29 Nov 2018 22:51:44 -0500 Subject: [PATCH 109/330] Remove use of std::tmpnam. (#734) std::tmpnam is deprecated and its use is discouraged. For our purposes in the tests, we really just need a file name which is unlikely to exist. This patch converts the tests to using a dummy random file name generator, which should hopefully avoid name conflicts. --- test/output_test_helper.cc | 42 +++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 6c9385ac0b..59c9dfb52a 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -438,11 +439,50 @@ int SubstrCnt(const std::string& haystack, const std::string& pat) { return count; } +static char ToHex(int ch) { + return ch < 10 ? static_cast('0' + ch) + : static_cast('a' + (ch - 10)); +} + +static char RandomHexChar() { + static std::mt19937 rd{std::random_device{}()}; + static std::uniform_int_distribution mrand{0, 15}; + return ToHex(mrand(rd)); +} + +static std::string GetRandomFileName() { + std::string model = "test.%%%%%%"; + for (auto & ch : model) { + if (ch == '%') + ch = RandomHexChar(); + } + return model; +} + +static bool FileExists(std::string const& name) { + std::ifstream in(name.c_str()); + return in.good(); +} + +static std::string GetTempFileName() { + // This function attempts to avoid race conditions where two tests + // create the same file at the same time. However, it still introduces races + // similar to tmpnam. + int retries = 3; + while (--retries) { + std::string name = GetRandomFileName(); + if (!FileExists(name)) + return name; + } + std::cerr << "Failed to create unique temporary file name" << std::endl; + std::abort(); +} + std::string GetFileReporterOutput(int argc, char* argv[]) { std::vector new_argv(argv, argv + argc); assert(static_cast(argc) == new_argv.size()); - std::string tmp_file_name = std::tmpnam(nullptr); + std::string tmp_file_name = GetTempFileName(); std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; std::string tmp = "--benchmark_out="; From eee8b05c97d7b832bf67d6e000958d012ab30165 Mon Sep 17 00:00:00 2001 From: Jusufadis Bakamovic Date: Fri, 7 Dec 2018 14:34:00 +0100 Subject: [PATCH 110/330] [tools] Run autopep8 and apply fixes found. (#739) --- tools/compare.py | 6 ++---- tools/gbench/report.py | 7 +++---- tools/gbench/util.py | 15 ++++++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/compare.py b/tools/compare.py index 9ff5c1424d..539ace6fb1 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import unittest """ compare.py - versatile benchmark output compare tool """ @@ -244,9 +245,6 @@ def main(): print(ln) -import unittest - - class TestParser(unittest.TestCase): def setUp(self): self.parser = create_parser() @@ -402,7 +400,7 @@ def test_benchmarksfiltered_with_remainder_after_doubleminus(self): if __name__ == '__main__': - #unittest.main() + # unittest.main() main() # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 28bab34af7..5085b93194 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -1,3 +1,4 @@ +import unittest """report.py - Utilities for reporting statistics about benchmark results """ import os @@ -270,9 +271,6 @@ def get_color(res): # Unit tests -import unittest - - class TestGetUniqueBenchmarkNames(unittest.TestCase): def load_results(self): import json @@ -290,7 +288,7 @@ def test_basic(self): 'BM_One', 'BM_Two', 'short', # These two are not sorted - 'medium', # These two are not sorted + 'medium', # These two are not sorted ] json = self.load_results() output_lines = get_unique_benchmark_names(json) @@ -300,6 +298,7 @@ def test_basic(self): for i in range(0, len(output_lines)): self.assertEqual(expect_lines[i], output_lines[i]) + class TestReportDifference(unittest.TestCase): def load_results(self): import json diff --git a/tools/gbench/util.py b/tools/gbench/util.py index 07c2377275..1f8e8e2c47 100644 --- a/tools/gbench/util.py +++ b/tools/gbench/util.py @@ -7,11 +7,13 @@ import sys # Input file type enumeration -IT_Invalid = 0 -IT_JSON = 1 +IT_Invalid = 0 +IT_JSON = 1 IT_Executable = 2 _num_magic_bytes = 2 if sys.platform.startswith('win') else 4 + + def is_executable_file(filename): """ Return 'True' if 'filename' names a valid file which is likely @@ -46,7 +48,7 @@ def is_json_file(filename): with open(filename, 'r') as f: json.load(f) return True - except: + except BaseException: pass return False @@ -84,6 +86,7 @@ def check_input_file(filename): sys.exit(1) return ftype + def find_benchmark_flag(prefix, benchmark_flags): """ Search the specified list of flags for a flag matching `` and @@ -97,6 +100,7 @@ def find_benchmark_flag(prefix, benchmark_flags): result = f[len(prefix):] return result + def remove_benchmark_flags(prefix, benchmark_flags): """ Return a new list containing the specified benchmark_flags except those @@ -105,6 +109,7 @@ def remove_benchmark_flags(prefix, benchmark_flags): assert prefix.startswith('--') and prefix.endswith('=') return [f for f in benchmark_flags if not f.startswith(prefix)] + def load_benchmark_results(fname): """ Read benchmark output from a file and return the JSON object. @@ -129,7 +134,7 @@ def run_benchmark(exe_name, benchmark_flags): thandle, output_name = tempfile.mkstemp() os.close(thandle) benchmark_flags = list(benchmark_flags) + \ - ['--benchmark_out=%s' % output_name] + ['--benchmark_out=%s' % output_name] cmd = [exe_name] + benchmark_flags print("RUNNING: %s" % ' '.join(cmd)) @@ -156,4 +161,4 @@ def run_or_load_benchmark(filename, benchmark_flags): elif ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) else: - assert False # This branch is unreachable \ No newline at end of file + assert False # This branch is unreachable From 5cb8f8a03d63f157e47c4146dec40433d1046bb3 Mon Sep 17 00:00:00 2001 From: Cyrille Date: Mon, 10 Dec 2018 11:24:22 +0100 Subject: [PATCH 111/330] Fix signed vs unsigned comparisons in string_util unit tests (#742) Unit-tests fail to build due to the following errors: /home/cfx/Dev/google-benchmark/benchmark.git/test/string_util_gtest.cc:12:5: required from here /home/cfx/Applications/googletest-1.8.1/include/gtest/gtest.h:1444:11: error: comparison between signed and unsigned integer expressions [-Werror=sign-compare] if (lhs == rhs) { ~~~~^~~~~~ Fixes #741 --- CONTRIBUTORS | 1 + test/string_util_gtest.cc | 62 +++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index f727bd1d08..ee74ff886c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -27,6 +27,7 @@ Arne Beer Billy Robert O'Neal III Chris Kennelly Christopher Seymour +Cyrille Faucheux David Coeurjolly Deniz Evrenci Dominic Hamon diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 4c81734cf8..2c5d073f61 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -9,56 +9,56 @@ namespace { TEST(StringUtilTest, stoul) { { size_t pos = 0; - EXPECT_EQ(0, benchmark::stoul("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(0ul, benchmark::stoul("0", &pos)); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; - EXPECT_EQ(7, benchmark::stoul("7", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(7ul, benchmark::stoul("7", &pos)); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; - EXPECT_EQ(135, benchmark::stoul("135", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(135ul, benchmark::stoul("135", &pos)); + EXPECT_EQ(3ul, pos); } #if ULONG_MAX == 0xFFFFFFFFul { size_t pos = 0; EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos)); - EXPECT_EQ(10, pos); + EXPECT_EQ(10ul, pos); } #elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul { size_t pos = 0; EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, benchmark::stoul("18446744073709551615", &pos)); - EXPECT_EQ(20, pos); + EXPECT_EQ(20ul, pos); } #endif { size_t pos = 0; - EXPECT_EQ(10, benchmark::stoul("1010", &pos, 2)); - EXPECT_EQ(4, pos); + EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(520, benchmark::stoul("1010", &pos, 8)); - EXPECT_EQ(4, pos); + EXPECT_EQ(520ul, benchmark::stoul("1010", &pos, 8)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(1010, benchmark::stoul("1010", &pos, 10)); - EXPECT_EQ(4, pos); + EXPECT_EQ(1010ul, benchmark::stoul("1010", &pos, 10)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(4112, benchmark::stoul("1010", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4112ul, benchmark::stoul("1010", &pos, 16)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(0xBEEF, benchmark::stoul("BEEF", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); + EXPECT_EQ(4ul, pos); } { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); @@ -69,42 +69,42 @@ TEST(StringUtilTest, stoi) { { size_t pos = 0; EXPECT_EQ(0, benchmark::stoi("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); @@ -115,28 +115,28 @@ TEST(StringUtilTest, stod) { { size_t pos = 0; EXPECT_EQ(0.0, benchmark::stod("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; /* Note: exactly representable as double */ EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); - EXPECT_EQ(8, pos); + EXPECT_EQ(8ul, pos); } { ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); From 1f3cba06e4f50bc35aa24941b58ae41fc2ada40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Ulvg=C3=A5rd?= Date: Mon, 10 Dec 2018 16:15:34 +0100 Subject: [PATCH 112/330] Update reference to complexity calculation (#723) --- src/complexity.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index 0be73e0822..6ef17660c9 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -73,8 +73,8 @@ std::string GetBigOString(BigO complexity) { // - time : Vector containing the times for the benchmark tests. // - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). -// For a deeper explanation on the algorithm logic, look the README file at -// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit +// For a deeper explanation on the algorithm logic, please refer to +// https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, From 47a5f77d754892f20b8c717ead26de836e0bb552 Mon Sep 17 00:00:00 2001 From: Jatin Chaudhary Date: Tue, 11 Dec 2018 16:53:02 +0530 Subject: [PATCH 113/330] #722 Adding Host Name in Reporting (#733) * Adding Host Name and test * Addressing Review Comments * Adding Test for JSON Reporter * Adding HOST_NAME_MAX for MacOS systems * Adding Explaination for MacOS HOST_NAME_MAX Addition * Addressing Peer Review Comments * Adding codecvt in windows header guard * Changing name SystemInfo and adding empty message incase host name fetch fails * Adding Comment on Struct SystemInfo --- include/benchmark/benchmark.h | 10 +++++++++ src/json_reporter.cc | 2 ++ src/reporter.cc | 3 ++- src/sysinfo.cc | 38 +++++++++++++++++++++++++++++++++++ test/reporter_output_test.cc | 1 + 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 1f072b1352..a0fd7c6e1c 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1293,6 +1293,15 @@ struct CPUInfo { BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); }; +//Adding Struct for System Information +struct SystemInfo { + std::string name; + static const SystemInfo& Get(); + private: + SystemInfo(); + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); +}; + // Interface for custom benchmark result printers. // By default, benchmark reports are printed to stdout. However an application // can control the destination of the reports by calling @@ -1302,6 +1311,7 @@ class BenchmarkReporter { public: struct Context { CPUInfo const& cpu_info; + SystemInfo const& sys_info; // The number of chars in the longest benchmark name. size_t name_field_width; static const char* executable_name; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index f599425a83..7d01e8e4e3 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -77,6 +77,8 @@ bool JSONReporter::ReportContext(const Context& context) { std::string walltime_value = LocalDateTimeString(); out << indent << FormatKV("date", walltime_value) << ",\n"; + out << indent << FormatKV("host_name", context.sys_info.name) << ",\n"; + if (Context::executable_name) { // windows uses backslash for its path separator, // which must be escaped in JSON otherwise it blows up conforming JSON diff --git a/src/reporter.cc b/src/reporter.cc index 056561de8c..59bc5f7102 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -79,7 +79,8 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, // No initializer because it's already initialized to NULL. const char *BenchmarkReporter::Context::executable_name; -BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()) {} +BenchmarkReporter::Context::Context() + : cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {} std::string BenchmarkReporter::Run::benchmark_name() const { std::string name = run_name; diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 0ad300ee0e..c0c07e5e62 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -19,6 +19,7 @@ #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include #include +#include #else #include #ifndef BENCHMARK_OS_FUCHSIA @@ -52,6 +53,7 @@ #include #include #include +#include #include "check.h" #include "cycleclock.h" @@ -366,6 +368,35 @@ std::vector GetCacheSizes() { #endif } +std::string GetSystemName() { +#if defined(BENCHMARK_OS_WINDOWS) + std::string str; + const unsigned COUNT = MAX_COMPUTERNAME_LENGTH+1; + TCHAR hostname[COUNT] = {'\0'}; + DWORD DWCOUNT = COUNT; + if (!GetComputerName(hostname, &DWCOUNT)) + return std::string(""); +#ifndef UNICODE + str = std::string(hostname, DWCOUNT); +#else + //Using wstring_convert, Is deprecated in C++17 + using convert_type = std::codecvt_utf8; + std::wstring_convert converter; + std::wstring wStr(hostname, DWCOUNT); + str = converter.to_bytes(wStr); +#endif + return str; +#else // defined(BENCHMARK_OS_WINDOWS) +#ifdef BENCHMARK_OS_MACOSX //Mac Doesnt have HOST_NAME_MAX defined +#define HOST_NAME_MAX 64 +#endif + char hostname[HOST_NAME_MAX]; + int retVal = gethostname(hostname, HOST_NAME_MAX); + if (retVal != 0) return std::string(""); + return std::string(hostname); +#endif // Catch-all POSIX block. +} + int GetNumCPUs() { #ifdef BENCHMARK_HAS_SYSCTL int NumCPU = -1; @@ -609,4 +640,11 @@ CPUInfo::CPUInfo() scaling_enabled(CpuScalingEnabled(num_cpus)), load_avg(GetLoadAvg()) {} + +const SystemInfo& SystemInfo::Get() { + static const SystemInfo* info = new SystemInfo(); + return *info; +} + +SystemInfo::SystemInfo() : name(GetSystemName()) {} } // end namespace benchmark diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 35bb9b26e9..8263831b92 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -23,6 +23,7 @@ static int AddContextCases() { {{"^\\{", MR_Default}, {"\"context\":", MR_Next}, {"\"date\": \"", MR_Next}, + {"\"host_name\":", MR_Next}, {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",", MR_Next}, {"\"num_cpus\": %int,$", MR_Next}, From 0ed529a7e3701539ebf12b206670cb9707ba185f Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 13 Dec 2018 11:14:50 +0000 Subject: [PATCH 114/330] Update documentation of benchmark_filter (#744) It should now match reality. --- src/benchmark.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 410493fded..aab07500af 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -57,9 +57,9 @@ DEFINE_bool(benchmark_list_tests, false, DEFINE_string(benchmark_filter, ".", "A regular expression that specifies the set of benchmarks " - "to execute. If this flag is empty, no benchmarks are run. " - "If this flag is the string \"all\", all benchmarks linked " - "into the process are run."); + "to execute. If this flag is empty, or if this flag is the " + "string \"all\", all benchmarks linked into the binary are " + "run."); DEFINE_double(benchmark_min_time, 0.5, "Minimum number of seconds we should run benchmark before " From 57bf879d494f4b070f4a403754bb4fa077977dcc Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Fri, 14 Dec 2018 01:20:01 +0300 Subject: [PATCH 115/330] README.md: document State::{Pause,Resume}Timing() As pointed out in IRC, these are not documented. --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index e4d5bf133c..858ea2334e 100644 --- a/README.md +++ b/README.md @@ -428,6 +428,26 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. +## Controlling timers +Normally, the entire duration of the work loop (`for (auto _ : state) {}`) +is measured. But sometimes, it is nessesary to do some work inside of +that loop, every iteration, but without counting that time to the benchmark time. +That is possible, althought it is not recommended, since it has high overhead. + +```c++ +static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { + std::set data; + for (auto _ : state) { + state.PauseTiming(); // Stop timers. They will not count until they are resumed. + data = ConstructRandomSet(state.range(0)); // Do something that should not be measured + state.ResumeTiming(); // And resume timers. They are now counting again. + // The rest will be measured. + for (int j = 0; j < state.range(1); ++j) + data.insert(RandomNumber()); + } +} +BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); +``` ## Manual timing For benchmarking something for which neither CPU time nor real-time are From 4528c76b718acc9b57956f63069c699ae21edcab Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 13 Dec 2018 22:49:21 -0500 Subject: [PATCH 116/330] Print at least three significant digits for times. (#701) Some benchmarks are particularly sensitive and they run in less than a nanosecond. In order for the console reporter to provide meaningful output for such benchmarks it needs to be able to display the times using more resolution than a single nanosecond. This patch changes the console reporter to print at least three significant digits for all results. Unlike the initial attempt, this patch does not align the decimal point. --- src/console_reporter.cc | 30 ++++++++++++++++++++++++------ test/output_test_helper.cc | 11 ++++++----- test/reporter_output_test.cc | 2 +- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/console_reporter.cc b/src/console_reporter.cc index 7de71386fe..ca364727cb 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -53,7 +53,7 @@ bool ConsoleReporter::ReportContext(const Context& context) { } void ConsoleReporter::PrintHeader(const Run& run) { - std::string str = FormatString("%-*s %13s %13s %10s", static_cast(name_field_width_), + std::string str = FormatString("%-*s %13s %15s %12s", static_cast(name_field_width_), "Benchmark", "Time", "CPU", "Iterations"); if(!run.counters.empty()) { if(output_options_ & OO_Tabular) { @@ -98,6 +98,21 @@ static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, va_end(args); } + +static std::string FormatTime(double time) { + // Align decimal places... + if (time < 1.0) { + return FormatString("%10.3f", time); + } + if (time < 10.0) { + return FormatString("%10.2f", time); + } + if (time < 100.0) { + return FormatString("%10.1f", time); + } + return FormatString("%10.0f", time); +} + void ConsoleReporter::PrintRunData(const Run& result) { typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); @@ -117,18 +132,21 @@ void ConsoleReporter::PrintRunData(const Run& result) { const double real_time = result.GetAdjustedRealTime(); const double cpu_time = result.GetAdjustedCPUTime(); + const std::string real_time_str = FormatTime(real_time); + const std::string cpu_time_str = FormatTime(cpu_time); + if (result.report_big_o) { std::string big_o = GetBigOString(result.complexity); - printer(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, big_o.c_str(), + printer(Out, COLOR_YELLOW, "%10.2f %-4s %10.2f %-4s ", real_time, big_o.c_str(), cpu_time, big_o.c_str()); } else if (result.report_rms) { - printer(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", real_time * 100, - cpu_time * 100); + printer(Out, COLOR_YELLOW, "%10.0f %-4s %10.0f %-4s ", real_time * 100, "%", + cpu_time * 100, "%"); } else { const char* timeLabel = GetTimeUnitString(result.time_unit); - printer(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", real_time, timeLabel, - cpu_time, timeLabel); + printer(Out, COLOR_YELLOW, "%s %-4s %s %-4s ", real_time_str.c_str(), timeLabel, + cpu_time_str.c_str(), timeLabel); } if (!result.report_big_o && !result.report_rms) { diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 59c9dfb52a..5dc951d2bc 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -39,17 +39,18 @@ SubMap& GetSubstitutions() { // Don't use 'dec_re' from header because it may not yet be initialized. // clang-format off static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; + static std::string time_re = "([0-9]+[.])?[0-9]+"; static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, - {"%time", "[ ]*[0-9]+ ns"}, - {"%console_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns [ ]*[0-9]+"}, - {"%console_time_only_report", "[ ]*[0-9]+ ns [ ]*[0-9]+ ns"}, - {"%console_us_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us [ ]*[0-9]+"}, - {"%console_us_time_only_report", "[ ]*[0-9]+ us [ ]*[0-9]+ us"}, + {"%time", "[ ]*" + time_re + "[ ]+ns"}, + {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, + {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, + {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, + {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, {"%csv_header", "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "items_per_second,label,error_occurred,error_message"}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8263831b92..ec6d51b359 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -534,7 +534,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " {"^BM_UserStats/iterations:5/repeats:3/" "manual_time_median [ ]* 150 ns %time [ ]*3$"}, {"^BM_UserStats/iterations:5/repeats:3/" - "manual_time_stddev [ ]* 0 ns %time [ ]*3$"}, + "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"}, {"^BM_UserStats/iterations:5/repeats:3/manual_time_ " "[ ]* 150 ns %time [ ]*3$"}}); ADD_CASES( From dc1064554905deca47386bdfc4b72f44c1bd7716 Mon Sep 17 00:00:00 2001 From: Reid Kleckner Date: Sun, 30 Dec 2018 09:46:07 -0800 Subject: [PATCH 117/330] Set CMP0048 policy before project() to silence warnings (#750) If this policy isn't set, CMake emits a large warning when project() is called from a cmake subdirectory. This came up when the benchmark library was added to the LLVM build, and it was reported in https://llvm.org/PR38874. This patch was the fix I applied locally to fix the issue, and I wanted to send it upstream. --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 310c7ee9f6..76e5fc5951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,7 @@ cmake_minimum_required (VERSION 2.8.12) -project (benchmark) - foreach(p + CMP0048 # OK to clear PROJECT_VERSION on project() CMP0054 # CMake 3.1 CMP0056 # export EXE_LINKER_FLAGS to try_run CMP0057 # Support no if() IN_LIST operator @@ -12,6 +11,8 @@ foreach(p endif() endforeach() +project (benchmark) + option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF) From eec9a8e4976a397988c15e5a4b71a542375a2240 Mon Sep 17 00:00:00 2001 From: "Michael \"Croydon\" Keck" Date: Thu, 3 Jan 2019 14:42:07 +0100 Subject: [PATCH 118/330] Add minimal Conan support (#728) --- conan/CMakeLists.txt | 7 +++ conan/test_package/CMakeLists.txt | 10 ++++ conan/test_package/conanfile.py | 19 +++++++ conan/test_package/test_package.cpp | 18 +++++++ conanfile.py | 79 +++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+) create mode 100644 conan/CMakeLists.txt create mode 100644 conan/test_package/CMakeLists.txt create mode 100644 conan/test_package/conanfile.py create mode 100644 conan/test_package/test_package.cpp create mode 100644 conanfile.py diff --git a/conan/CMakeLists.txt b/conan/CMakeLists.txt new file mode 100644 index 0000000000..15b92ca91a --- /dev/null +++ b/conan/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 2.8.11) +project(cmake_wrapper) + +include(conanbuildinfo.cmake) +conan_basic_setup() + +include(${CMAKE_SOURCE_DIR}/CMakeListsOriginal.txt) diff --git a/conan/test_package/CMakeLists.txt b/conan/test_package/CMakeLists.txt new file mode 100644 index 0000000000..089a6c729d --- /dev/null +++ b/conan/test_package/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.11) +project(test_package) + +set(CMAKE_VERBOSE_MAKEFILE TRUE) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_basic_setup() + +add_executable(${PROJECT_NAME} test_package.cpp) +target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) diff --git a/conan/test_package/conanfile.py b/conan/test_package/conanfile.py new file mode 100644 index 0000000000..d63f4088c9 --- /dev/null +++ b/conan/test_package/conanfile.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from conans import ConanFile, CMake +import os + + +class TestPackageConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "cmake" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + bin_path = os.path.join("bin", "test_package") + self.run(bin_path, run_environment=True) diff --git a/conan/test_package/test_package.cpp b/conan/test_package/test_package.cpp new file mode 100644 index 0000000000..4fa7ec0bf9 --- /dev/null +++ b/conan/test_package/test_package.cpp @@ -0,0 +1,18 @@ +#include "benchmark/benchmark.h" + +void BM_StringCreation(benchmark::State& state) { + while (state.KeepRunning()) + std::string empty_string; +} + +BENCHMARK(BM_StringCreation); + +void BM_StringCopy(benchmark::State& state) { + std::string x = "hello"; + while (state.KeepRunning()) + std::string copy(x); +} + +BENCHMARK(BM_StringCopy); + +BENCHMARK_MAIN(); diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000000..e31fc5268a --- /dev/null +++ b/conanfile.py @@ -0,0 +1,79 @@ +from conans import ConanFile, CMake, tools +from conans.errors import ConanInvalidConfiguration +import shutil +import os + + +class GoogleBenchmarkConan(ConanFile): + name = "benchmark" + description = "A microbenchmark support library." + topics = ("conan", "benchmark", "google", "microbenchmark") + url = "https://github.com/google/benchmark" + homepage = "https://github.com/google/benchmark" + author = "Google Inc." + license = "Apache-2.0" + exports_sources = ["*"] + generators = "cmake" + + settings = "arch", "build_type", "compiler", "os" + options = { + "shared": [True, False], + "fPIC": [True, False], + "enable_lto": [True, False], + "enable_exceptions": [True, False] + } + default_options = {"shared": False, "fPIC": True, "enable_lto": False, "enable_exceptions": True} + + _build_subfolder = "." + + def source(self): + # Wrap the original CMake file to call conan_basic_setup + shutil.move("CMakeLists.txt", "CMakeListsOriginal.txt") + shutil.move(os.path.join("conan", "CMakeLists.txt"), "CMakeLists.txt") + + def config_options(self): + if self.settings.os == "Windows": + if self.settings.compiler == "Visual Studio" and float(self.settings.compiler.version.value) <= 12: + raise ConanInvalidConfiguration("{} {} does not support Visual Studio <= 12".format(self.name, self.version)) + del self.options.fPIC + + def configure(self): + if self.settings.os == "Windows" and self.options.shared: + raise ConanInvalidConfiguration("Windows shared builds are not supported right now, see issue #639") + + def _configure_cmake(self): + cmake = CMake(self) + + cmake.definitions["BENCHMARK_ENABLE_TESTING"] = "OFF" + cmake.definitions["BENCHMARK_ENABLE_GTEST_TESTS"] = "OFF" + cmake.definitions["BENCHMARK_ENABLE_LTO"] = "ON" if self.options.enable_lto else "OFF" + cmake.definitions["BENCHMARK_ENABLE_EXCEPTIONS"] = "ON" if self.options.enable_exceptions else "OFF" + + # See https://github.com/google/benchmark/pull/638 for Windows 32 build explanation + if self.settings.os != "Windows": + cmake.definitions["BENCHMARK_BUILD_32_BITS"] = "ON" if "64" not in str(self.settings.arch) else "OFF" + cmake.definitions["BENCHMARK_USE_LIBCXX"] = "ON" if (str(self.settings.compiler.libcxx) == "libc++") else "OFF" + else: + cmake.definitions["BENCHMARK_USE_LIBCXX"] = "OFF" + + cmake.configure(build_folder=self._build_subfolder) + return cmake + + def build(self): + cmake = self._configure_cmake() + cmake.build() + + def package(self): + cmake = self._configure_cmake() + cmake.install() + + self.copy(pattern="LICENSE", dst="licenses") + + def package_info(self): + self.cpp_info.libs = tools.collect_libs(self) + if self.settings.os == "Linux": + self.cpp_info.libs.extend(["pthread", "rt"]) + elif self.settings.os == "Windows": + self.cpp_info.libs.append("shlwapi") + elif self.settings.os == "SunOS": + self.cpp_info.libs.append("kstat") From 4b9f43e2c4ea559c59f5d86561c02207008a15ac Mon Sep 17 00:00:00 2001 From: Andriy Berestovskyy Date: Sun, 13 Jan 2019 15:26:49 +0100 Subject: [PATCH 119/330] Fix header lines length (#752) Commit 17a012d7 added a newline to the str, so the line built from str.length() is one character longer than it should be. --- AUTHORS | 1 + CONTRIBUTORS | 1 + src/console_reporter.cc | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 09e2e0551a..ef3051a15b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -9,6 +9,7 @@ # Please keep the list sorted. Albert Pretorius +Andriy Berestovskyy Arne Beer Carto Christopher Seymour diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ee74ff886c..d0c31df16f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -23,6 +23,7 @@ # Please keep the list sorted. Albert Pretorius +Andriy Berestovskyy Arne Beer Billy Robert O'Neal III Chris Kennelly diff --git a/src/console_reporter.cc b/src/console_reporter.cc index ca364727cb..cc8ae276f6 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -64,9 +64,8 @@ void ConsoleReporter::PrintHeader(const Run& run) { str += " UserCounters..."; } } - str += "\n"; std::string line = std::string(str.length(), '-'); - GetOutputStream() << line << "\n" << str << line << "\n"; + GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; } void ConsoleReporter::ReportRuns(const std::vector& reports) { From 785e2c3158589e8ef48c59ba80e48d76bdbd8902 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 15 Jan 2019 13:19:36 +0000 Subject: [PATCH 120/330] Move Statistics struct to internal namespace (#753) --- include/benchmark/benchmark.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index a0fd7c6e1c..e4f39211a9 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -421,6 +421,7 @@ typedef double(BigOFunc)(int64_t); // statistics over all the measurements of some type typedef double(StatisticsFunc)(const std::vector&); +namespace internal { struct Statistics { std::string name_; StatisticsFunc* compute_; @@ -429,7 +430,6 @@ struct Statistics { : name_(name), compute_(compute) {} }; -namespace internal { struct BenchmarkInstance; class ThreadTimer; class ThreadManager; @@ -1373,7 +1373,7 @@ class BenchmarkReporter { int64_t complexity_n; // what statistics to compute from the measurements - const std::vector* statistics; + const std::vector* statistics; // Inform print function whether the current run is a complexity report bool report_big_o; From 97393e5ef820fda0330830d56c774949b4cfbb29 Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Fri, 1 Feb 2019 13:51:44 +0100 Subject: [PATCH 121/330] Add -lpthread to pkg-config Libs.private. (#755) Since pthread is required at least for GCC (according to the documentation), this should be reflected by the pkg-config file. The same is, for instance, also done by the gflags library: https://github.com/gflags/gflags/blob/1005485222e8b0feff822c5723ddcaa5abadc01a/cmake/package.pc.in#L13 --- cmake/benchmark.pc.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/benchmark.pc.in b/cmake/benchmark.pc.in index 1e84bff68d..43ca8f91d7 100644 --- a/cmake/benchmark.pc.in +++ b/cmake/benchmark.pc.in @@ -8,4 +8,5 @@ Description: Google microbenchmark framework Version: @VERSION@ Libs: -L${libdir} -lbenchmark +Libs.private: -lpthread Cflags: -I${includedir} From b8ca0c42179b7b5d656494e61dda8b861057122f Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 4 Feb 2019 16:26:51 +0300 Subject: [PATCH 122/330] README.md: mention that fixture has SetUp() / TearDown() --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 858ea2334e..902915eb9d 100644 --- a/README.md +++ b/README.md @@ -615,7 +615,14 @@ creating/registering the tests using the following macros: For Example: ```c++ -class MyFixture : public benchmark::Fixture {}; +class MyFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) { + } + + void TearDown(const ::benchmark::State& state) { + } +}; BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { for (auto _ : st) { From 7c571338b5cb13c2e3f5ce9106175c49c149c4a5 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 19 Feb 2019 06:32:11 -0600 Subject: [PATCH 123/330] Prefer -pthread to -lpthread for better compatibility when cross-compiling (#771) --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 76e5fc5951..a61600923a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,6 +251,7 @@ if (NOT BENCHMARK_ENABLE_EXCEPTIONS AND HAVE_STD_REGEX endif() cxx_feature_check(STEADY_CLOCK) # Ensure we have pthreads +set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) # Set up directories From 0ae233ab23c560547bf85ce1346580966e799861 Mon Sep 17 00:00:00 2001 From: Jilin Zhou Date: Tue, 19 Feb 2019 08:05:55 -0500 Subject: [PATCH 124/330] [#766] add x-compile support for QNX SDP7 (#770) Since googletest already supports x-compilation for QNX, it is nice to have google benchmark support it too. Fixes #766 --- CMakeLists.txt | 4 ++++ src/internal_macros.h | 2 ++ src/sysinfo.cc | 2 ++ 3 files changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a61600923a..d7ed57e0a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,6 +183,10 @@ else() add_definitions(-D_GNU_SOURCE=1) endif() + if (QNXNTO) + add_definitions(-D_QNX_SOURCE) + endif() + # Link time optimisation if (BENCHMARK_ENABLE_LTO) add_cxx_compiler_flag(-flto) diff --git a/src/internal_macros.h b/src/internal_macros.h index 5dbf4fd275..6adf00d056 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -70,6 +70,8 @@ #define BENCHMARK_OS_FUCHSIA 1 #elif defined (__SVR4) && defined (__sun) #define BENCHMARK_OS_SOLARIS 1 +#elif defined(__QNX__) +#define BENCHMARK_OS_QNX 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index c0c07e5e62..55953a05ce 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -389,6 +389,8 @@ std::string GetSystemName() { #else // defined(BENCHMARK_OS_WINDOWS) #ifdef BENCHMARK_OS_MACOSX //Mac Doesnt have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 +#elif defined(BENCHMARK_OS_QNX) +#define HOST_NAME_MAX 154 #endif char hostname[HOST_NAME_MAX]; int retVal = gethostname(hostname, HOST_NAME_MAX); From d205ead299c7cddd5e1bc3478d57ad4320a4a53c Mon Sep 17 00:00:00 2001 From: Jilin Zhou Date: Thu, 28 Feb 2019 05:42:44 -0500 Subject: [PATCH 125/330] [#774] implement GetNumCPUs(), GetCPUCyclesPerSecond(), and GetCacheSizes() (#775) - On qnx platform, cpu and cache info is stored in a syspage struct which is different from other OS platform. - The fix has been verified on an aarch64 target running qnx 7.0. Fixes #774 --- src/sysinfo.cc | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 55953a05ce..953c170cfd 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -37,6 +37,9 @@ #if defined(BENCHMARK_OS_SOLARIS) #include #endif +#if defined(BENCHMARK_OS_QNX) +#include +#endif #include #include @@ -209,6 +212,9 @@ bool ReadFromFile(std::string const& fname, ArgT* arg) { bool CpuScalingEnabled(int num_cpus) { // We don't have a valid CPU count, so don't even bother. if (num_cpus <= 0) return false; +#ifdef BENCHMARK_OS_QNX + return false; +#endif #ifndef BENCHMARK_OS_WINDOWS // On Linux, the CPUfreq subsystem exposes CPU information as files on the // local file system. If reading the exported files fails, then we may not be @@ -356,6 +362,40 @@ std::vector GetCacheSizesWindows() { } return res; } +#elif BENCHMARK_OS_QNX +std::vector GetCacheSizesQNX() { + std::vector res; + struct cacheattr_entry *cache = SYSPAGE_ENTRY(cacheattr); + uint32_t const elsize = SYSPAGE_ELEMENT_SIZE(cacheattr); + int num = SYSPAGE_ENTRY_SIZE(cacheattr) / elsize ; + for(int i = 0; i < num; ++i ) { + CPUInfo::CacheInfo info; + switch (cache->flags){ + case CACHE_FLAG_INSTR : + info.type = "Instruction"; + info.level = 1; + break; + case CACHE_FLAG_DATA : + info.type = "Data"; + info.level = 1; + break; + case CACHE_FLAG_UNIFIED : + info.type = "Unified"; + info.level = 2; + case CACHE_FLAG_SHARED : + info.type = "Shared"; + info.level = 3; + default : + continue; + break; + } + info.size = cache->line_size * cache->num_lines; + info.num_sharing = 0; + res.push_back(std::move(info)); + cache = SYSPAGE_ARRAY_ADJ_OFFSET(cacheattr, cache, elsize); + } + return res; +} #endif std::vector GetCacheSizes() { @@ -363,6 +403,8 @@ std::vector GetCacheSizes() { return GetCacheSizesMacOSX(); #elif defined(BENCHMARK_OS_WINDOWS) return GetCacheSizesWindows(); +#elif defined(BENCHMARK_OS_QNX) + return GetCacheSizesQNX(); #else return GetCacheSizesFromKVFS(); #endif @@ -423,6 +465,8 @@ int GetNumCPUs() { strerror(errno)); } return NumCPU; +#elif defined(BENCHMARK_OS_QNX) + return static_cast(_syspage_ptr->num_cpu); #else int NumCPUs = 0; int MaxID = -1; @@ -602,6 +646,9 @@ double GetCPUCyclesPerSecond() { double clock_hz = knp->value.ui64; kstat_close(kc); return clock_hz; +#elif defined (BENCHMARK_OS_QNX) + return static_cast((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * + (int64_t)(1000 * 1000)); #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. const int estimate_time_ms = 1000; From f62c63b14f87688f0d66deafecafb1a8a2872e0c Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 6 Mar 2019 17:22:28 +0300 Subject: [PATCH 126/330] [Tooling] report.py: don't crash on BigO/RMS benchmarks Run into this by accident while writing benchmark to validate the fix for https://bugs.llvm.org/show_bug.cgi?id=40965 Fixes #779. --- tools/gbench/Inputs/test1_run1.json | 17 +++++++++++++++++ tools/gbench/Inputs/test1_run2.json | 17 +++++++++++++++++ tools/gbench/report.py | 15 +++++++++++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json index d7ec6a9c8f..b07db7c89a 100644 --- a/tools/gbench/Inputs/test1_run1.json +++ b/tools/gbench/Inputs/test1_run1.json @@ -97,6 +97,23 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" + }, + { + "name": "MyComplexityTest_BigO", + "run_name": "MyComplexityTest", + "run_type": "aggregate", + "aggregate_name": "BigO", + "cpu_coefficient": 4.2749856294592886e+00, + "real_coefficient": 6.4789275289789780e+00, + "big_o": "N", + "time_unit": "ns" + }, + { + "name": "MyComplexityTest_RMS", + "run_name": "MyComplexityTest", + "run_type": "aggregate", + "aggregate_name": "RMS", + "rms": 4.5097802512472874e-03 } ] } diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json index 59a5ffaca4..8b7e8ecad3 100644 --- a/tools/gbench/Inputs/test1_run2.json +++ b/tools/gbench/Inputs/test1_run2.json @@ -97,6 +97,23 @@ "real_time": 1, "cpu_time": 1, "time_unit": "ns" + }, + { + "name": "MyComplexityTest_BigO", + "run_name": "MyComplexityTest", + "run_type": "aggregate", + "aggregate_name": "BigO", + "cpu_coefficient": 5.6215779594361486e+00, + "real_coefficient": 5.6288314793554610e+00, + "big_o": "N", + "time_unit": "ns" + }, + { + "name": "MyComplexityTest_RMS", + "run_name": "MyComplexityTest", + "run_type": "aggregate", + "aggregate_name": "RMS", + "rms": 3.3128901852342174e-03 } ] } diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 5085b93194..e1e1c14697 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -114,6 +114,10 @@ def intersect(list1, list2): return [x for x in list1 if x in list2] +def is_potentially_comparable_benchmark(x): + return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x) + + def partition_benchmarks(json1, json2): """ While preserving the ordering, find benchmarks with the same names in @@ -125,10 +129,17 @@ def partition_benchmarks(json1, json2): names = intersect(json1_unique_names, json2_unique_names) partitions = [] for name in names: + time_unit = None # Pick the time unit from the first entry of the lhs benchmark. - time_unit = (x['time_unit'] - for x in json1['benchmarks'] if x['name'] == name).next() + # We should be careful not to crash with unexpected input. + for x in json1['benchmarks']: + if (x['name'] == name and is_potentially_comparable_benchmark(x)): + time_unit = x['time_unit'] + break + if time_unit is None: + break # Filter by name and time unit. + # All the repetitions are assumed to be comparable. lhs = [x for x in json1['benchmarks'] if x['name'] == name and x['time_unit'] == time_unit] rhs = [x for x in json2['benchmarks'] if x['name'] == name and From df7c7ee1d37dda0fb597586b4624515166a778d0 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 6 Mar 2019 18:13:22 +0300 Subject: [PATCH 127/330] [Tooling] report.py: whoops, don't ignore the rest of benches after a bad one. Refs #779. --- tools/gbench/Inputs/test1_run1.json | 28 ++++++++++++++-------------- tools/gbench/Inputs/test1_run2.json | 28 ++++++++++++++-------------- tools/gbench/report.py | 4 ++-- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json index b07db7c89a..601e327aef 100644 --- a/tools/gbench/Inputs/test1_run1.json +++ b/tools/gbench/Inputs/test1_run1.json @@ -84,20 +84,6 @@ "cpu_time": 100, "time_unit": "ns" }, - { - "name": "BM_BadTimeUnit", - "iterations": 1000, - "real_time": 0.4, - "cpu_time": 0.5, - "time_unit": "s" - }, - { - "name": "BM_DifferentTimeUnit", - "iterations": 1, - "real_time": 1, - "cpu_time": 1, - "time_unit": "s" - }, { "name": "MyComplexityTest_BigO", "run_name": "MyComplexityTest", @@ -114,6 +100,20 @@ "run_type": "aggregate", "aggregate_name": "RMS", "rms": 4.5097802512472874e-03 + }, + { + "name": "BM_NotBadTimeUnit", + "iterations": 1000, + "real_time": 0.4, + "cpu_time": 0.5, + "time_unit": "s" + }, + { + "name": "BM_DifferentTimeUnit", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "s" } ] } diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json index 8b7e8ecad3..3cbcf39b0c 100644 --- a/tools/gbench/Inputs/test1_run2.json +++ b/tools/gbench/Inputs/test1_run2.json @@ -84,20 +84,6 @@ "cpu_time": 66.664, "time_unit": "ns" }, - { - "name": "BM_BadTimeUnit", - "iterations": 1000, - "real_time": 0.04, - "cpu_time": 0.6, - "time_unit": "s" - }, - { - "name": "BM_DifferentTimeUnit", - "iterations": 1, - "real_time": 1, - "cpu_time": 1, - "time_unit": "ns" - }, { "name": "MyComplexityTest_BigO", "run_name": "MyComplexityTest", @@ -114,6 +100,20 @@ "run_type": "aggregate", "aggregate_name": "RMS", "rms": 3.3128901852342174e-03 + }, + { + "name": "BM_NotBadTimeUnit", + "iterations": 1000, + "real_time": 0.04, + "cpu_time": 0.6, + "time_unit": "s" + }, + { + "name": "BM_DifferentTimeUnit", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "ns" } ] } diff --git a/tools/gbench/report.py b/tools/gbench/report.py index e1e1c14697..49b46eca65 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -137,7 +137,7 @@ def partition_benchmarks(json1, json2): time_unit = x['time_unit'] break if time_unit is None: - break + continue # Filter by name and time unit. # All the repetitions are assumed to be comparable. lhs = [x for x in json1['benchmarks'] if x['name'] == name and @@ -341,7 +341,7 @@ def test_basic(self): ['BM_10PercentCPUToTime', '+0.1000', '-0.1000', '100', '110', '100', '90'], ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], - ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], + ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ] json1, json2 = self.load_results() output_lines_with_header = generate_difference_report( From f6e96861a373c90ea0c727177fc68d2984e048bb Mon Sep 17 00:00:00 2001 From: Daniel Harvey Date: Sun, 17 Mar 2019 14:38:51 +0100 Subject: [PATCH 128/330] BENCHMARK_CAPTURE() and Complexity() - naming problem (#761) Created BenchmarkName class which holds the full benchmark name and allows specifying and retrieving different components of the name (e.g. ARGS, THREADS etc.) Fixes #730. --- AUTHORS | 1 + CONTRIBUTORS | 1 + include/benchmark/benchmark.h | 19 ++++++++- src/benchmark.cc | 5 ++- src/benchmark_api_internal.h | 2 +- src/benchmark_name.cc | 58 +++++++++++++++++++++++++++ src/benchmark_register.cc | 34 +++++++++------- src/benchmark_runner.cc | 2 +- src/complexity.cc | 5 ++- src/json_reporter.cc | 2 +- src/reporter.cc | 2 +- src/statistics.cc | 2 +- test/CMakeLists.txt | 1 + test/benchmark_name_gtest.cc | 74 +++++++++++++++++++++++++++++++++++ test/complexity_test.cc | 20 ++++++++++ 15 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 src/benchmark_name.cc create mode 100644 test/benchmark_name_gtest.cc diff --git a/AUTHORS b/AUTHORS index ef3051a15b..d5cd007895 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Andriy Berestovskyy Arne Beer Carto Christopher Seymour +Daniel Harvey David Coeurjolly Deniz Evrenci Dirac Research diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d0c31df16f..c86d873564 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -29,6 +29,7 @@ Billy Robert O'Neal III Chris Kennelly Christopher Seymour Cyrille Faucheux +Daniel Harvey David Coeurjolly Deniz Evrenci Dominic Hamon diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index e4f39211a9..2954ff480c 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1302,6 +1302,23 @@ struct SystemInfo { BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); }; +// BenchmarkName contains the components of the Benchmark's name +// which allows individual fields to be modified or cleared before +// building the final name using 'str()'. +struct BenchmarkName { + std::string function_name; + std::string args; + std::string min_time; + std::string iterations; + std::string repetitions; + std::string time_type; + std::string threads; + + // Return the full name of the benchmark with each non-empty + // field separated by a '/' + std::string str() const; +}; + // Interface for custom benchmark result printers. // By default, benchmark reports are printed to stdout. However an application // can control the destination of the reports by calling @@ -1340,7 +1357,7 @@ class BenchmarkReporter { max_bytes_used(0) {} std::string benchmark_name() const; - std::string run_name; + BenchmarkName run_name; RunType run_type; // is this a measurement, or an aggregate? std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. diff --git a/src/benchmark.cc b/src/benchmark.cc index aab07500af..c76346c6c2 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -233,7 +233,7 @@ void RunBenchmarks(const std::vector& benchmarks, size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { name_field_width = - std::max(name_field_width, benchmark.name.size()); + std::max(name_field_width, benchmark.name.str().size()); might_have_aggregates |= benchmark.repetitions > 1; for (const auto& Stat : *benchmark.statistics) @@ -393,7 +393,8 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } if (FLAGS_benchmark_list_tests) { - for (auto const& benchmark : benchmarks) Out << benchmark.name << "\n"; + for (auto const& benchmark : benchmarks) + Out << benchmark.name.str() << "\n"; } else { internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 0524a85c01..b220fb0d22 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -16,7 +16,7 @@ namespace internal { // Information kept per benchmark we may want to run struct BenchmarkInstance { - std::string name; + BenchmarkName name; Benchmark* benchmark; AggregationReportMode aggregation_report_mode; std::vector arg; diff --git a/src/benchmark_name.cc b/src/benchmark_name.cc new file mode 100644 index 0000000000..2a17ebce27 --- /dev/null +++ b/src/benchmark_name.cc @@ -0,0 +1,58 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace benchmark { + +namespace { + +// Compute the total size of a pack of std::strings +size_t size_impl() { return 0; } + +template +size_t size_impl(const Head& head, const Tail&... tail) { + return head.size() + size_impl(tail...); +} + +// Join a pack of std::strings using a delimiter +// TODO: use absl::StrJoin +void join_impl(std::string&, char) {} + +template +void join_impl(std::string& s, const char delimiter, const Head& head, + const Tail&... tail) { + if (!s.empty() && !head.empty()) { + s += delimiter; + } + + s += head; + + join_impl(s, delimiter, tail...); +} + +template +std::string join(char delimiter, const Ts&... ts) { + std::string s; + s.reserve(sizeof...(Ts) + size_impl(ts...)); + join_impl(s, delimiter, ts...); + return s; +} +} // namespace + +std::string BenchmarkName::str() const { + return join('/', function_name, args, min_time, iterations, repetitions, + time_type, threads); +} +} // namespace benchmark diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index f17f5b223c..9d77529797 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -153,7 +153,7 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { BenchmarkInstance instance; - instance.name = family->name_; + instance.name.function_name = family->name_; instance.benchmark = family.get(); instance.aggregation_report_mode = family->aggregation_report_mode_; instance.arg = args; @@ -172,45 +172,51 @@ bool BenchmarkFamilies::FindBenchmarks( // Add arguments to instance name size_t arg_i = 0; for (auto const& arg : args) { - instance.name += "/"; + if (!instance.name.args.empty()) { + instance.name.args += '/'; + } if (arg_i < family->arg_names_.size()) { const auto& arg_name = family->arg_names_[arg_i]; if (!arg_name.empty()) { - instance.name += - StrFormat("%s:", family->arg_names_[arg_i].c_str()); + instance.name.args += StrFormat("%s:", arg_name.c_str()); } } // we know that the args are always non-negative (see 'AddRange()'), // thus print as 'unsigned'. BUT, do a cast due to the 32-bit builds. - instance.name += StrFormat("%lu", static_cast(arg)); + instance.name.args += + StrFormat("%lu", static_cast(arg)); + ++arg_i; } if (!IsZero(family->min_time_)) - instance.name += StrFormat("/min_time:%0.3f", family->min_time_); + instance.name.min_time = + StrFormat("min_time:%0.3f", family->min_time_); if (family->iterations_ != 0) { - instance.name += - StrFormat("/iterations:%lu", + instance.name.iterations = + StrFormat("iterations:%lu", static_cast(family->iterations_)); } if (family->repetitions_ != 0) - instance.name += StrFormat("/repeats:%d", family->repetitions_); + instance.name.repetitions = + StrFormat("repeats:%d", family->repetitions_); if (family->use_manual_time_) { - instance.name += "/manual_time"; + instance.name.time_type = "manual_time"; } else if (family->use_real_time_) { - instance.name += "/real_time"; + instance.name.time_type = "real_time"; } // Add the number of threads used to the name if (!family->thread_counts_.empty()) { - instance.name += StrFormat("/threads:%d", instance.threads); + instance.name.threads = StrFormat("threads:%d", instance.threads); } - if ((re.Match(instance.name) && !isNegativeFilter) || - (!re.Match(instance.name) && isNegativeFilter)) { + const auto full_name = instance.name.str(); + if ((re.Match(full_name) && !isNegativeFilter) || + (!re.Match(full_name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 38faeec8e3..9e1f6e4617 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -191,7 +191,7 @@ class BenchmarkRunner { double seconds; }; IterationResults DoNIterations() { - VLOG(2) << "Running " << b.name << " for " << iters << "\n"; + VLOG(2) << "Running " << b.name.str() << " for " << iters << "\n"; std::unique_ptr manager; manager.reset(new internal::ThreadManager(b.threads)); diff --git a/src/complexity.cc b/src/complexity.cc index 6ef17660c9..4c6fcf6d2e 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -183,8 +183,9 @@ std::vector ComputeBigO( result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); } - std::string run_name = reports[0].benchmark_name().substr( - 0, reports[0].benchmark_name().find('/')); + // Drop the 'args' when reporting complexity. + auto run_name = reports[0].run_name; + run_name.args.clear(); // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 7d01e8e4e3..26f94c6beb 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -168,7 +168,7 @@ void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; - out << indent << FormatKV("run_name", run.run_name) << ",\n"; + out << indent << FormatKV("run_name", run.run_name.str()) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { case BenchmarkReporter::Run::RT_Iteration: diff --git a/src/reporter.cc b/src/reporter.cc index 59bc5f7102..4d3e477d44 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -83,7 +83,7 @@ BenchmarkReporter::Context::Context() : cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {} std::string BenchmarkReporter::Run::benchmark_name() const { - std::string name = run_name; + std::string name = run_name.str(); if (run_type == RT_Aggregate) { name += "_" + aggregate_name; } diff --git a/src/statistics.cc b/src/statistics.cc index e821aec18b..e9a170f17a 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -147,7 +147,7 @@ std::vector ComputeStats( for (const auto& Stat : *reports[0].statistics) { // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; - data.run_name = reports[0].benchmark_name(); + data.run_name = reports[0].run_name; data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.aggregate_name = Stat.name_; data.report_label = report_label; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f15ce20818..e7373b438e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -191,6 +191,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) endmacro() add_gtest(benchmark_gtest) + add_gtest(benchmark_name_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) diff --git a/test/benchmark_name_gtest.cc b/test/benchmark_name_gtest.cc new file mode 100644 index 0000000000..afb401c1f5 --- /dev/null +++ b/test/benchmark_name_gtest.cc @@ -0,0 +1,74 @@ +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" + +namespace { + +using namespace benchmark; +using namespace benchmark::internal; + +TEST(BenchmarkNameTest, Empty) { + const auto name = BenchmarkName(); + EXPECT_EQ(name.str(), std::string()); +} + +TEST(BenchmarkNameTest, FunctionName) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + EXPECT_EQ(name.str(), "function_name"); +} + +TEST(BenchmarkNameTest, FunctionNameAndArgs) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.args = "some_args:3/4/5"; + EXPECT_EQ(name.str(), "function_name/some_args:3/4/5"); +} + +TEST(BenchmarkNameTest, MinTime) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.args = "some_args:3/4"; + name.min_time = "min_time:3.4s"; + EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s"); +} + +TEST(BenchmarkNameTest, Iterations) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.min_time = "min_time:3.4s"; + name.iterations = "iterations:42"; + EXPECT_EQ(name.str(), "function_name/min_time:3.4s/iterations:42"); +} + +TEST(BenchmarkNameTest, Repetitions) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.min_time = "min_time:3.4s"; + name.repetitions = "repetitions:24"; + EXPECT_EQ(name.str(), "function_name/min_time:3.4s/repetitions:24"); +} + +TEST(BenchmarkNameTest, TimeType) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.min_time = "min_time:3.4s"; + name.time_type = "hammer_time"; + EXPECT_EQ(name.str(), "function_name/min_time:3.4s/hammer_time"); +} + +TEST(BenchmarkNameTest, Threads) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.min_time = "min_time:3.4s"; + name.threads = "threads:256"; + EXPECT_EQ(name.str(), "function_name/min_time:3.4s/threads:256"); +} + +TEST(BenchmarkNameTest, TestEmptyFunctionName) { + auto name = BenchmarkName(); + name.args = "first:3/second:4"; + name.threads = "threads:22"; + EXPECT_EQ(name.str(), "first:3/second:4/threads:22"); +} + +} // end namespace diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 323ddfe7ac..b0fd87116d 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -176,6 +176,26 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); +// ========================================================================= // +// -------- Testing formatting of Complexity with captured args ------------ // +// ========================================================================= // + +void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { + for (auto _ : state) { + } + state.SetComplexityN(n); +} + +BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) + ->Complexity(benchmark::oN) + ->Ranges({{1, 2}, {3, 4}}); + +const std::string complexity_capture_name = + "BM_ComplexityCaptureArgs/capture_test"; + +ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", + complexity_capture_name + "_RMS", "N"); + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // From 5acb0f05ed4938ac3140918048cfe1da2b5c6951 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sun, 17 Mar 2019 18:48:35 +0300 Subject: [PATCH 129/330] Travis-ci: fix clang+libc++ build (#783) It broke because the libc++ is being built as part of *this* build, with old gcc+libstdc++ (4.8?), but LLVM is preparing to switch to C++14, and gcc+libstdc++ <5 are soft-deprecated. Just the gcc update doesn't cut it, clang still uses old libstdc++. --- .travis.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4625dfb087..51d652cf98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,54 +42,64 @@ matrix: env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release # Clang w/ libc++ - compiler: clang + dist: xenial addons: apt: packages: clang-3.8 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - EXTRA_FLAGS="-stdlib=libc++" - compiler: clang + dist: xenial addons: apt: packages: clang-3.8 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - EXTRA_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang + dist: xenial addons: apt: packages: - clang-3.8 - g++-multilib env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ 32bit libc++ - compiler: clang + dist: xenial addons: apt: packages: - clang-3.8 - g++-multilib env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ libc++, ASAN, UBSAN - compiler: clang + dist: xenial addons: apt: packages: clang-3.8 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - ENABLE_SANITIZER=1 @@ -97,22 +107,26 @@ matrix: - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN - compiler: clang + dist: xenial addons: apt: packages: clang-3.8 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" # Clang w/ libc++ and MSAN - compiler: clang + dist: xenial addons: apt: packages: clang-3.8 env: + - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - ENABLE_SANITIZER=1 From fae87266906c10fc055eb270eddc622404696e63 Mon Sep 17 00:00:00 2001 From: Michael Tesch Date: Tue, 19 Mar 2019 11:12:54 +0100 Subject: [PATCH 130/330] Replace JSON inf and nan with JS compliant Infinity and NaN --- src/json_reporter.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 26f94c6beb..25beba019e 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include "string_util.h" #include "timers.h" @@ -53,10 +54,15 @@ std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << key << "\": "; - const auto max_digits10 = std::numeric_limits::max_digits10; - const auto max_fractional_digits10 = max_digits10 - 1; - - ss << std::scientific << std::setprecision(max_fractional_digits10) << value; + if (std::isnan(value)) + ss << "NaN"; + else if (std::isinf(value)) + ss << (value < 0 ? "-" : "") << "Infinity"; + else { + const auto max_digits10 = std::numeric_limits::max_digits10; + const auto max_fractional_digits10 = max_digits10 - 1; + ss << std::scientific << std::setprecision(max_fractional_digits10) << value; + } return ss.str(); } From 478eafa36bb8763e04c61d88bb2b8e9fa3440b82 Mon Sep 17 00:00:00 2001 From: BaaMeow <38274252+BaaMeow@users.noreply.github.com> Date: Tue, 26 Mar 2019 05:53:07 -0400 Subject: [PATCH 131/330] [JSON] add threads and repetitions to the json output (#748) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [JSON] add threads and repetitions to the json output, for better ide… [Tests] explicitly check for thread == 1 [Tests] specifically mark all repetition checks [JSON] add repetition_index reporting, but only for non-aggregates (i… * [Formatting] Be very, very explicit about pointer alignment so clang-format can not put pointers/references on the wrong side of arguments. [Benchmark::Run] Make sure to use explanatory sentinel variable rather than a magic number. * Do not pass redundant information --- .clang-format | 1 + .gitignore | 3 + include/benchmark/benchmark.h | 21 +++-- src/benchmark_runner.cc | 17 ++-- src/complexity.cc | 8 +- src/json_reporter.cc | 14 +++- src/statistics.cc | 3 + test/complexity_test.cc | 10 ++- test/memory_manager_test.cc | 6 +- test/reporter_output_test.cc | 115 ++++++++++++++++++++++++++- test/user_counters_tabular_test.cc | 15 ++++ test/user_counters_test.cc | 30 +++++++ test/user_counters_thousands_test.cc | 12 +++ 13 files changed, 230 insertions(+), 25 deletions(-) diff --git a/.clang-format b/.clang-format index 4b3f13fa55..70fe225d78 100644 --- a/.clang-format +++ b/.clang-format @@ -3,3 +3,4 @@ Language: Cpp BasedOnStyle: Google ... +PointerAlignment: Left diff --git a/.gitignore b/.gitignore index 8c30e28f53..806d04c6b3 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ build*/ # Visual Studio 2015/2017 cache/options directory .vs/ CMakeSettings.json + +# Visual Studio Code cache/options directory +.vscode/ diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 2954ff480c..51e15dc51a 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -246,11 +246,11 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #endif #if defined(__GNUC__) || __has_builtin(__builtin_unreachable) - #define BENCHMARK_UNREACHABLE() __builtin_unreachable() +#define BENCHMARK_UNREACHABLE() __builtin_unreachable() #elif defined(_MSC_VER) - #define BENCHMARK_UNREACHABLE() __assume(false) +#define BENCHMARK_UNREACHABLE() __assume(false) #else - #define BENCHMARK_UNREACHABLE() ((void)0) +#define BENCHMARK_UNREACHABLE() ((void)0) #endif namespace benchmark { @@ -1293,10 +1293,11 @@ struct CPUInfo { BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo); }; -//Adding Struct for System Information +// Adding Struct for System Information struct SystemInfo { std::string name; static const SystemInfo& Get(); + private: SystemInfo(); BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo); @@ -1336,12 +1337,14 @@ class BenchmarkReporter { }; struct Run { + static const int64_t no_repetition_index = -1; enum RunType { RT_Iteration, RT_Aggregate }; Run() : run_type(RT_Iteration), error_occurred(false), iterations(1), + threads(1), time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), @@ -1358,13 +1361,16 @@ class BenchmarkReporter { std::string benchmark_name() const; BenchmarkName run_name; - RunType run_type; // is this a measurement, or an aggregate? + RunType run_type; std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. bool error_occurred; std::string error_message; int64_t iterations; + int64_t threads; + int64_t repetition_index; + int64_t repetitions; TimeUnit time_unit; double real_accumulated_time; double cpu_accumulated_time; @@ -1502,8 +1508,9 @@ class JSONReporter : public BenchmarkReporter { bool first_report_; }; -class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future release") - CSVReporter : public BenchmarkReporter { +class BENCHMARK_DEPRECATED_MSG( + "The CSV Reporter will be removed in a future release") CSVReporter + : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} virtual bool ReportContext(const Context& context); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 9e1f6e4617..f4ea5f27f3 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -64,7 +64,8 @@ static const size_t kMaxIterations = 1000000000; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, const internal::ThreadManager::Result& results, size_t memory_iterations, - const MemoryManager::Result& memory_result, double seconds) { + const MemoryManager::Result& memory_result, double seconds, + int64_t repetition_index) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -75,6 +76,9 @@ BenchmarkReporter::Run CreateRunReport( // This is the total iterations across all threads. report.iterations = results.iterations; report.time_unit = b.time_unit; + report.threads = b.threads; + report.repetition_index = repetition_index; + report.repetitions = b.repetitions; if (!report.error_occurred) { if (b.use_manual_time) { @@ -150,8 +154,7 @@ class BenchmarkRunner { } for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { - const bool is_the_first_repetition = repetition_num == 0; - DoOneRepetition(is_the_first_repetition); + DoOneRepetition(repetition_num); } // Calculate additional statistics @@ -276,7 +279,8 @@ class BenchmarkRunner { ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time); } - void DoOneRepetition(bool is_the_first_repetition) { + void DoOneRepetition(int64_t repetition_index) { + const bool is_the_first_repetition = repetition_index == 0; IterationResults i; // We *may* be gradually increasing the length (iteration count) @@ -326,8 +330,9 @@ class BenchmarkRunner { } // Ok, now actualy report. - BenchmarkReporter::Run report = CreateRunReport( - b, i.results, memory_iterations, memory_result, i.seconds); + BenchmarkReporter::Run report = + CreateRunReport(b, i.results, memory_iterations, memory_result, + i.seconds, repetition_index); if (!report.error_occurred && b.complexity != oNone) complexity_reports.push_back(report); diff --git a/src/complexity.cc b/src/complexity.cc index 4c6fcf6d2e..e65bd2ebdf 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -191,7 +191,11 @@ std::vector ComputeBigO( Run big_o; big_o.run_name = run_name; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; + big_o.repetitions = reports[0].repetitions; + big_o.repetition_index = Run::no_repetition_index; + big_o.threads = reports[0].threads; big_o.aggregate_name = "BigO"; + big_o.report_label = reports[0].report_label; big_o.iterations = 0; big_o.real_accumulated_time = result_real.coef; big_o.cpu_accumulated_time = result_cpu.coef; @@ -208,11 +212,13 @@ std::vector ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; rms.run_name = run_name; - big_o.report_label = reports[0].report_label; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; rms.iterations = 0; + rms.repetition_index = Run::no_repetition_index; + rms.repetitions = reports[0].repetitions; + rms.threads = reports[0].threads; rms.real_accumulated_time = result_real.rms / multiplier; rms.cpu_accumulated_time = result_cpu.rms / multiplier; rms.report_rms = true; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 25beba019e..dd4ba30e38 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -16,6 +16,7 @@ #include "complexity.h" #include +#include #include #include // for setprecision #include @@ -23,7 +24,6 @@ #include #include #include -#include #include "string_util.h" #include "timers.h" @@ -59,9 +59,11 @@ std::string FormatKV(std::string const& key, double value) { else if (std::isinf(value)) ss << (value < 0 ? "-" : "") << "Infinity"; else { - const auto max_digits10 = std::numeric_limits::max_digits10; + const auto max_digits10 = + std::numeric_limits::max_digits10; const auto max_fractional_digits10 = max_digits10 - 1; - ss << std::scientific << std::setprecision(max_fractional_digits10) << value; + ss << std::scientific << std::setprecision(max_fractional_digits10) + << value; } return ss.str(); } @@ -184,6 +186,12 @@ void JSONReporter::PrintRunData(Run const& run) { } BENCHMARK_UNREACHABLE(); }()) << ",\n"; + out << indent << FormatKV("repetitions", run.repetitions) << ",\n"; + if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) { + out << indent << FormatKV("repetition_index", run.repetition_index) + << ",\n"; + } + out << indent << FormatKV("threads", run.threads) << ",\n"; if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) { out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n"; } diff --git a/src/statistics.cc b/src/statistics.cc index e9a170f17a..4dc2c95f31 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -149,6 +149,9 @@ std::vector ComputeStats( Run data; data.run_name = reports[0].run_name; data.run_type = BenchmarkReporter::Run::RT_Aggregate; + data.threads = reports[0].threads; + data.repetitions = reports[0].repetitions; + data.repetition_index = Run::no_repetition_index; data.aggregate_name = Stat.name_; data.report_label = report_label; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index b0fd87116d..4a62869248 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -28,6 +28,8 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, @@ -37,6 +39,8 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, {"\"name\": \"%rms_name\",$"}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"RMS\",$", MR_Next}, {"\"rms\": %float$", MR_Next}, {"}", MR_Next}}); @@ -156,7 +160,9 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) { return kLog2E * n * log(static_cast(n)); }); + ->Complexity([](int64_t n) { + return kLog2E * n * log(static_cast(n)); + }); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -180,7 +186,7 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, // -------- Testing formatting of Complexity with captured args ------------ // // ========================================================================= // -void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { +void BM_ComplexityCaptureArgs(benchmark::State& state, int n) { for (auto _ : state) { } state.SetComplexityN(n); diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 94be608379..90bed16cff 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -23,6 +23,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -32,8 +35,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); - -int main(int argc, char *argv[]) { +int main(int argc, char* argv[]) { std::unique_ptr mm(new TestMemoryManager()); benchmark::RegisterMemoryManager(mm.get()); diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index ec6d51b359..8b7ae935aa 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -74,6 +74,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -97,6 +100,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -121,6 +127,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -144,6 +153,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -167,6 +179,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, {"\"error_message\": \"message\",$", MR_Next}}); @@ -185,7 +200,10 @@ BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // @@ -200,7 +218,10 @@ BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); // ========================================================================= // @@ -217,7 +238,10 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); // ========================================================================= // @@ -267,22 +291,34 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, @@ -302,25 +338,40 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -342,28 +393,46 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"repetition_index\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"repetition_index\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 4,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -384,7 +453,10 @@ BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, - {"\"run_type\": \"iteration\",$", MR_Next}}); + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); // Test that non-aggregate data is not reported @@ -404,16 +476,22 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, @@ -440,16 +518,22 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, @@ -480,18 +564,24 @@ ADD_CASES(TC_JSONOut, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}}); @@ -543,24 +633,35 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, @@ -568,6 +669,8 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, @@ -575,6 +678,8 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -582,6 +687,8 @@ ADD_CASES( {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 030e98916c..099464ef99 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -73,6 +73,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -121,6 +124,9 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -168,6 +174,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -203,6 +212,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -242,6 +254,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index bb0d6b4c5a..0775bc01f7 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -34,6 +34,9 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -75,6 +78,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -117,6 +123,9 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -152,6 +161,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -188,6 +200,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -226,6 +241,9 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -262,6 +280,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -303,6 +324,9 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -342,6 +366,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -380,6 +407,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index fa0ef97204..21d8285ded 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -53,6 +53,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -67,6 +70,9 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -81,6 +87,8 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -96,6 +104,8 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, @@ -111,6 +121,8 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, {"\"aggregate_name\": \"stddev\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, From e3666568a90b4f0e72b85252e5a64d9f1bcfca0a Mon Sep 17 00:00:00 2001 From: Daniel Harvey Date: Tue, 26 Mar 2019 11:50:53 +0100 Subject: [PATCH 132/330] Negative ranges #762 (#787) * Add FIXME in multiple_ranges_test.cc * Improve handling of large bounds in AddRange. Due to breaking the loop too early, AddRange would miss a final multplier of 'mult' that was within the numeric range of T. * Enable negative values for Range argument Fixes #762. * Try to fix build of benchmark_gtest * Try some more to fix build * Attempt to fix format macros * Attempt to resolve format errors for mingw32 * Review feedback Put unit tests in benchmark::internal namespace Fix error reporting in multiple_ranges_test.cc --- src/benchmark_register.cc | 10 ++-- src/benchmark_register.h | 92 ++++++++++++++++++++++++++++++---- src/string_util.h | 4 +- test/benchmark_gtest.cc | 97 +++++++++++++++++++++++++++++++++++- test/multiple_ranges_test.cc | 3 +- test/options_test.cc | 10 ++++ 6 files changed, 197 insertions(+), 19 deletions(-) diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 9d77529797..4e94b5a219 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -34,6 +34,9 @@ #include #include +#define __STDC_FORMAT_MACROS +#include + #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" #include "check.h" @@ -183,11 +186,7 @@ bool BenchmarkFamilies::FindBenchmarks( } } - // we know that the args are always non-negative (see 'AddRange()'), - // thus print as 'unsigned'. BUT, do a cast due to the 32-bit builds. - instance.name.args += - StrFormat("%lu", static_cast(arg)); - + instance.name.args += StrFormat("%" PRId64, arg); ++arg_i; } @@ -334,7 +333,6 @@ Benchmark* Benchmark::ArgNames(const std::vector& names) { Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); - CHECK_GE(start, 0); CHECK_LE(start, limit); for (int64_t arg = start; arg <= limit; arg += step) { args_.push_back({arg}); diff --git a/src/benchmark_register.h b/src/benchmark_register.h index 0705e219f2..61377d7423 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -5,29 +5,103 @@ #include "check.h" +namespace benchmark { +namespace internal { + +// Append the powers of 'mult' in the closed interval [lo, hi]. +// Returns iterator to the start of the inserted range. template -void AddRange(std::vector* dst, T lo, T hi, int mult) { +typename std::vector::iterator +AddPowers(std::vector* dst, T lo, T hi, int mult) { CHECK_GE(lo, 0); CHECK_GE(hi, lo); CHECK_GE(mult, 2); - // Add "lo" - dst->push_back(lo); + const size_t start_offset = dst->size(); static const T kmax = std::numeric_limits::max(); - // Now space out the benchmarks in multiples of "mult" - for (T i = 1; i < kmax / mult; i *= mult) { - if (i >= hi) break; - if (i > lo) { + // Space out the values in multiples of "mult" + for (T i = 1; i <= hi; i *= mult) { + if (i >= lo) { dst->push_back(i); } + // Break the loop here since multiplying by + // 'mult' would move outside of the range of T + if (i > kmax / mult) break; + } + + return dst->begin() + start_offset; +} + +template +void AddNegatedPowers(std::vector* dst, T lo, T hi, int mult) { + // We negate lo and hi so we require that they cannot be equal to 'min'. + CHECK_GT(lo, std::numeric_limits::min()); + CHECK_GT(hi, std::numeric_limits::min()); + CHECK_GE(hi, lo); + CHECK_LE(hi, 0); + + // Add positive powers, then negate and reverse. + // Casts necessary since small integers get promoted + // to 'int' when negating. + const auto lo_complement = static_cast(-lo); + const auto hi_complement = static_cast(-hi); + + const auto it = AddPowers(dst, hi_complement, lo_complement, mult); + + std::for_each(it, dst->end(), [](T& t) { t *= -1; }); + std::reverse(it, dst->end()); +} + +template +void AddRange(std::vector* dst, T lo, T hi, int mult) { + static_assert(std::is_integral::value && std::is_signed::value, + "Args type must be a signed integer"); + + CHECK_GE(hi, lo); + CHECK_GE(mult, 2); + + // Add "lo" + dst->push_back(lo); + + // Handle lo == hi as a special case, so we then know + // lo < hi and so it is safe to add 1 to lo and subtract 1 + // from hi without falling outside of the range of T. + if (lo == hi) return; + + // Ensure that lo_inner <= hi_inner below. + if (lo + 1 == hi) { + dst->push_back(hi); + return; } - // Add "hi" (if different from "lo") - if (hi != lo) { + // Add all powers of 'mult' in the range [lo+1, hi-1] (inclusive). + const auto lo_inner = static_cast(lo + 1); + const auto hi_inner = static_cast(hi - 1); + + // Insert negative values + if (lo_inner < 0) { + AddNegatedPowers(dst, lo_inner, std::min(hi_inner, T{-1}), mult); + } + + // Treat 0 as a special case (see discussion on #762). + if (lo <= 0 && hi >= 0) { + dst->push_back(0); + } + + // Insert positive values + if (hi_inner > 0) { + AddPowers(dst, std::max(lo_inner, T{1}), hi_inner, mult); + } + + // Add "hi" (if different from last value). + if (hi != dst->back()) { dst->push_back(hi); } } +} // namespace internal +} // namespace benchmark + #endif // BENCHMARK_REGISTER_H diff --git a/src/string_util.h b/src/string_util.h index fc5f8b0304..c5dcee6fcd 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -12,7 +12,9 @@ void AppendHumanReadable(int n, std::string* str); std::string HumanReadableNumber(double n, double one_k = 1024.0); -#ifdef __GNUC__ +#if defined(__MINGW32__) +__attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) +#elif defined(__GNUC__) __attribute__((format(printf, 1, 2))) #endif std::string diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index 10683b433a..9557b20ec7 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -4,6 +4,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +namespace benchmark { +namespace internal { namespace { TEST(AddRangeTest, Simple) { @@ -30,4 +32,97 @@ TEST(AddRangeTest, Advanced64) { EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15)); } -} // end namespace +TEST(AddRangeTest, FullRange8) { + std::vector dst; + AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8); + EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); +} + +TEST(AddRangeTest, FullRange64) { + std::vector dst; + AddRange(&dst, int64_t{1}, std::numeric_limits::max(), 1024); + EXPECT_THAT( + dst, testing::ElementsAre(1LL, 1024LL, 1048576LL, 1073741824LL, + 1099511627776LL, 1125899906842624LL, + 1152921504606846976LL, 9223372036854775807LL)); +} + +TEST(AddRangeTest, NegativeRanges) { + std::vector dst; + AddRange(&dst, -8, 0, 2); + EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0)); +} + +TEST(AddRangeTest, StrictlyNegative) { + std::vector dst; + AddRange(&dst, -8, -1, 2); + EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1)); +} + +TEST(AddRangeTest, SymmetricNegativeRanges) { + std::vector dst; + AddRange(&dst, -8, 8, 2); + EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0, 1, 2, 4, 8)); +} + +TEST(AddRangeTest, SymmetricNegativeRangesOddMult) { + std::vector dst; + AddRange(&dst, -30, 32, 5); + EXPECT_THAT(dst, testing::ElementsAre(-30, -25, -5, -1, 0, 1, 5, 25, 32)); +} + +TEST(AddRangeTest, NegativeRangesAsymmetric) { + std::vector dst; + AddRange(&dst, -3, 5, 2); + EXPECT_THAT(dst, testing::ElementsAre(-3, -2, -1, 0, 1, 2, 4, 5)); +} + +TEST(AddRangeTest, NegativeRangesLargeStep) { + // Always include -1, 0, 1 when crossing zero. + std::vector dst; + AddRange(&dst, -8, 8, 10); + EXPECT_THAT(dst, testing::ElementsAre(-8, -1, 0, 1, 8)); +} + +TEST(AddRangeTest, ZeroOnlyRange) { + std::vector dst; + AddRange(&dst, 0, 0, 2); + EXPECT_THAT(dst, testing::ElementsAre(0)); +} + +TEST(AddRangeTest, NegativeRange64) { + std::vector dst; + AddRange(&dst, -4, 4, 2); + EXPECT_THAT(dst, testing::ElementsAre(-4, -2, -1, 0, 1, 2, 4)); +} + +TEST(AddRangeTest, NegativeRangePreservesExistingOrder) { + // If elements already exist in the range, ensure we don't change + // their ordering by adding negative values. + std::vector dst = {1, 2, 3}; + AddRange(&dst, -2, 2, 2); + EXPECT_THAT(dst, testing::ElementsAre(1, 2, 3, -2, -1, 0, 1, 2)); +} + +TEST(AddRangeTest, FullNegativeRange64) { + std::vector dst; + const auto min = std::numeric_limits::min(); + const auto max = std::numeric_limits::max(); + AddRange(&dst, min, max, 1024); + EXPECT_THAT( + dst, testing::ElementsAreArray(std::vector{ + min, -1152921504606846976LL, -1125899906842624LL, + -1099511627776LL, -1073741824LL, -1048576LL, -1024LL, -1LL, 0LL, + 1LL, 1024LL, 1048576LL, 1073741824LL, 1099511627776LL, + 1125899906842624LL, 1152921504606846976LL, max})); +} + +TEST(AddRangeTest, Simple8) { + std::vector dst; + AddRange(&dst, 1, 8, 2); + EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); +} + +} // namespace +} // namespace internal +} // namespace benchmark diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc index c64acabc25..b25f40eb52 100644 --- a/test/multiple_ranges_test.cc +++ b/test/multiple_ranges_test.cc @@ -40,8 +40,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. virtual ~MultipleRangesFixture() { - assert(actualValues.size() == expectedValues.size()); - if (actualValues.size() != expectedValues.size()) { + if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (auto v : expectedValues) { std::cout << "{"; diff --git a/test/options_test.cc b/test/options_test.cc index fdec69174e..7bfc235465 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -35,6 +35,16 @@ BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); BENCHMARK(BM_basic)->Repetitions(3); +BENCHMARK(BM_basic) + ->RangeMultiplier(std::numeric_limits::max()) + ->Range(std::numeric_limits::min(), + std::numeric_limits::max()); + +// Negative ranges +BENCHMARK(BM_basic)->Range(-64, -1); +BENCHMARK(BM_basic)->RangeMultiplier(4)->Range(-8, 8); +BENCHMARK(BM_basic)->DenseRange(-2, 2, 1); +BENCHMARK(BM_basic)->Ranges({{-64, 1}, {-8, -1}}); void CustomArgs(benchmark::internal::Benchmark* b) { for (int i = 0; i < 10; ++i) { From 64dcec387a76f98066ee43e85a2ba3cb52f48941 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 28 Mar 2019 10:49:40 +0000 Subject: [PATCH 133/330] Set theme jekyll-theme-midnight --- _config.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 _config.yml diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000000..18854876c6 --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-midnight \ No newline at end of file From 6a5c379cafef3b49314b600bec6a928ebb2f209a Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 28 Mar 2019 10:53:47 +0000 Subject: [PATCH 134/330] Set theme jekyll-theme-midnight --- docs/_config.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/_config.yml diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000000..18854876c6 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-midnight \ No newline at end of file From 3bc802e47c430b144123160484d00adbbe70214d Mon Sep 17 00:00:00 2001 From: Joseph Loser Date: Fri, 5 Apr 2019 04:43:47 -0600 Subject: [PATCH 135/330] Silence CMake Policy 0063 warning (#790) Summary: - When google benchmark is used as a submodule in a parent projects whose min CMake version is 3.3.2 or later, the google benchmark `CMakeLists.txt` triggers a warning regarding CMake policy 0063: ``` CMake Warning (dev) at tests/googlebenchmark/src/CMakeLists.txt:19 (add_library): Policy CMP0063 is not set: Honor visibility properties for all target types. Run "cmake --help-policy CMP0063" for policy details. Use the cmake_policy command to set the policy and suppress this warning. Target "benchmark" of type "STATIC_LIBRARY" has the following visibility properties set for CXX: CXX_VISIBILITY_PRESET VISIBILITY_INLINES_HIDDEN For compatibility CMake is not honoring them for this target. This warning is for project developers. Use -Wno-dev to suppress it. ``` - Set CMake Policy 0063 to NEW if the policy is available. This will not affect parent projects who include benchmark but do not have a CMake min version of 3.3.2 or later, i.e. when this policy is introduced. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d7ed57e0a5..e51cb38533 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,7 @@ foreach(p CMP0054 # CMake 3.1 CMP0056 # export EXE_LINKER_FLAGS to try_run CMP0057 # Support no if() IN_LIST operator + CMP0063 # Honor visibility properties for all targets ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) From 30bd6ea7f8c678ec105a65add605a0c3c1107a44 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 8 Apr 2019 12:38:11 +0300 Subject: [PATCH 136/330] Fix .clang-format --- .clang-format | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.clang-format b/.clang-format index 70fe225d78..e7d00feaa0 100644 --- a/.clang-format +++ b/.clang-format @@ -1,6 +1,5 @@ --- Language: Cpp BasedOnStyle: Google -... - PointerAlignment: Left +... From 7a1c37028359ca9d386d719a6ad527743cf1b753 Mon Sep 17 00:00:00 2001 From: Bryan Lunt Date: Tue, 9 Apr 2019 07:01:33 -0500 Subject: [PATCH 137/330] Add process_time for better OpenMP and user-managed thread timing * Google Benchmark now works with OpenMP and other user-managed threading. --- README.md | 43 ++++++++ include/benchmark/benchmark.h | 18 +++- src/benchmark_api_internal.h | 1 + src/benchmark_register.cc | 22 +++- src/benchmark_runner.cc | 7 +- src/thread_timer.h | 23 +++- test/CMakeLists.txt | 3 + test/internal_threading_test.cc | 184 ++++++++++++++++++++++++++++++++ 8 files changed, 290 insertions(+), 11 deletions(-) create mode 100644 test/internal_threading_test.cc diff --git a/README.md b/README.md index 902915eb9d..11d7a8b347 100644 --- a/README.md +++ b/README.md @@ -428,6 +428,49 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. +## CPU timers +By default, the CPU timer only measures the time spent by the main thread. +If the benchmark itself uses threads internally, this measurement may not +be what you are looking for. Instead, there is a way to measure the total +CPU usage of the process, by all the threads. + +```c++ +void callee(int i); + +static void MyMain(int size) { +#pragma omp parallel for + for(int i = 0; i < size; i++) + callee(i); +} + +static void BM_OpenMP(benchmark::State& state) { + for (auto _ : state) + MyMain(state.range(0); +} + +// Measure the time spent by the main thread, use it to decide for how long to +// run the benchmark loop. Depending on the internal implementation detail may +// measure to anywhere from near-zero (the overhead spent before/after work +// handoff to worker thread[s]) to the whole single-thread time. +BENCHMARK(BM_OpenMP)->Range(8, 8<<10); + +// Measure the user-visible time, the wall clock (literally, the time that +// has passed on the clock on the wall), use it to decide for how long to +// run the benchmark loop. This will always be meaningful, an will match the +// time spent by the main thread in single-threaded case, in general decreasing +// with the number of internal threads doing the work. +BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); + +// Measure the total CPU consumption, use it to decide for how long to +// run the benchmark loop. This will always measure to no less than the +// time spent by the main thread in single-threaded case. +BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime(); + +// A mixture of the last two. Measure the total CPU consumption, but use the +// wall clock to decide for how long to run the benchmark loop. +BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime(); +``` + ## Controlling timers Normally, the entire duration of the work loop (`for (auto _ : state) {}`) is measured. But sometimes, it is nessesary to do some work inside of diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 51e15dc51a..8329368571 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -874,11 +874,18 @@ class Benchmark { // Same as ReportAggregatesOnly(), but applies to display reporter only. Benchmark* DisplayAggregatesOnly(bool value = true); - // If a particular benchmark is I/O bound, runs multiple threads internally or - // if for some reason CPU timings are not representative, call this method. If - // called, the elapsed time will be used to control how many iterations are - // run, and in the printing of items/second or MB/seconds values. If not - // called, the cpu time used by the benchmark will be used. + // By default, the CPU time is measured only for the main thread, which may + // be unrepresentative if the benchmark uses threads internally. If called, + // the total CPU time spent by all the threads will be measured instead. + // By default, the only the main thread CPU time will be measured. + Benchmark* MeasureProcessCPUTime(); + + // If a particular benchmark should use the Wall clock instead of the CPU time + // (be it either the CPU time of the main thread only (default), or the + // total CPU usage of the benchmark), call this method. If called, the elapsed + // (wall) time will be used to control how many iterations are run, and in the + // printing of items/second or MB/seconds values. + // If not called, the CPU time used by the benchmark will be used. Benchmark* UseRealTime(); // If a benchmark must measure time manually (e.g. if GPU execution time is @@ -952,6 +959,7 @@ class Benchmark { double min_time_; size_t iterations_; int repetitions_; + bool measure_process_cpu_time_; bool use_real_time_; bool use_manual_time_; BigO complexity_; diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index b220fb0d22..19c6d4f7d2 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -22,6 +22,7 @@ struct BenchmarkInstance { std::vector arg; TimeUnit time_unit; int range_multiplier; + bool measure_process_cpu_time; bool use_real_time; bool use_manual_time; BigO complexity; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 4e94b5a219..3ffd734550 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -165,6 +165,7 @@ bool BenchmarkFamilies::FindBenchmarks( instance.min_time = family->min_time_; instance.iterations = family->iterations_; instance.repetitions = family->repetitions_; + instance.measure_process_cpu_time = family->measure_process_cpu_time_; instance.use_real_time = family->use_real_time_; instance.use_manual_time = family->use_manual_time_; instance.complexity = family->complexity_; @@ -202,10 +203,20 @@ bool BenchmarkFamilies::FindBenchmarks( instance.name.repetitions = StrFormat("repeats:%d", family->repetitions_); + if (family->measure_process_cpu_time_) { + instance.name.time_type = "process_time"; + } + if (family->use_manual_time_) { - instance.name.time_type = "manual_time"; + if (!instance.name.time_type.empty()) { + instance.name.time_type += '/'; + } + instance.name.time_type += "manual_time"; } else if (family->use_real_time_) { - instance.name.time_type = "real_time"; + if (!instance.name.time_type.empty()) { + instance.name.time_type += '/'; + } + instance.name.time_type += "real_time"; } // Add the number of threads used to the name @@ -252,6 +263,7 @@ Benchmark::Benchmark(const char* name) min_time_(0), iterations_(0), repetitions_(0), + measure_process_cpu_time_(false), use_real_time_(false), use_manual_time_(false), complexity_(oNone), @@ -398,6 +410,12 @@ Benchmark* Benchmark::DisplayAggregatesOnly(bool value) { return this; } +Benchmark* Benchmark::MeasureProcessCPUTime() { + // Can be used together with UseRealTime() / UseManualTime(). + measure_process_cpu_time_ = true; + return this; +} + Benchmark* Benchmark::UseRealTime() { CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index f4ea5f27f3..b1c0b8896a 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -111,7 +111,10 @@ BenchmarkReporter::Run CreateRunReport( // Adds the stats collected for the thread into *total. void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, ThreadManager* manager) { - internal::ThreadTimer timer; + internal::ThreadTimer timer( + b->measure_process_cpu_time + ? internal::ThreadTimer::CreateProcessCpuTime() + : internal::ThreadTimer::Create()); State st = b->Run(iters, thread_id, &timer, manager); CHECK(st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; @@ -226,6 +229,8 @@ class BenchmarkRunner { // Adjust real/manual time stats since they were reported per thread. i.results.real_time_used /= b.threads; i.results.manual_time_used /= b.threads; + // If we were measuring whole-process CPU usage, adjust the CPU time too. + if (b.measure_process_cpu_time) i.results.cpu_time_used /= b.threads; VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; diff --git a/src/thread_timer.h b/src/thread_timer.h index eaf108e017..fbd298d3bd 100644 --- a/src/thread_timer.h +++ b/src/thread_timer.h @@ -8,14 +8,22 @@ namespace benchmark { namespace internal { class ThreadTimer { + explicit ThreadTimer(bool measure_process_cpu_time_) + : measure_process_cpu_time(measure_process_cpu_time_) {} + public: - ThreadTimer() = default; + static ThreadTimer Create() { + return ThreadTimer(/*measure_process_cpu_time_=*/false); + } + static ThreadTimer CreateProcessCpuTime() { + return ThreadTimer(/*measure_process_cpu_time_=*/true); + } // Called by each thread void StartTimer() { running_ = true; start_real_time_ = ChronoClockNow(); - start_cpu_time_ = ThreadCPUUsage(); + start_cpu_time_ = ReadCpuTimerOfChoice(); } // Called by each thread @@ -25,7 +33,8 @@ class ThreadTimer { real_time_used_ += ChronoClockNow() - start_real_time_; // Floating point error can result in the subtraction producing a negative // time. Guard against that. - cpu_time_used_ += std::max(ThreadCPUUsage() - start_cpu_time_, 0); + cpu_time_used_ += + std::max(ReadCpuTimerOfChoice() - start_cpu_time_, 0); } // Called by each thread @@ -52,6 +61,14 @@ class ThreadTimer { } private: + double ReadCpuTimerOfChoice() const { + if (measure_process_cpu_time) return ProcessCPUUsage(); + return ThreadCPUUsage(); + } + + // should the thread, or the process, time be measured? + const bool measure_process_cpu_time; + bool running_ = false; // Is the timer running double start_real_time_ = 0; // If running_ double start_cpu_time_ = 0; // If running_ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e7373b438e..62370a54eb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -125,6 +125,9 @@ add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01 compile_output_test(user_counters_test) add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) +compile_output_test(internal_threading_test) +add_test(internal_threading_test internal_threading_test --benchmark_min_time=0.01) + compile_output_test(report_aggregates_only_test) add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01) diff --git a/test/internal_threading_test.cc b/test/internal_threading_test.cc new file mode 100644 index 0000000000..039d7c14a8 --- /dev/null +++ b/test/internal_threading_test.cc @@ -0,0 +1,184 @@ + +#undef NDEBUG + +#include +#include +#include "../src/timers.h" +#include "benchmark/benchmark.h" +#include "output_test.h" + +static const std::chrono::duration time_frame(50); +static const double time_frame_in_sec( + std::chrono::duration_cast>>( + time_frame) + .count()); + +void MyBusySpinwait() { + const auto start = benchmark::ChronoClockNow(); + + while (true) { + const auto now = benchmark::ChronoClockNow(); + const auto elapsed = now - start; + + if (std::chrono::duration(elapsed) >= + time_frame) + return; + } +} + +// ========================================================================= // +// --------------------------- TEST CASES BEGIN ---------------------------- // +// ========================================================================= // + +// ========================================================================= // +// BM_MainThread + +void BM_MainThread(benchmark::State& state) { + for (auto _ : state) { + MyBusySpinwait(); + state.SetIterationTime(time_frame_in_sec); + } + state.counters["invtime"] = + benchmark::Counter{1, benchmark::Counter::kIsRate}; +} + +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseRealTime(); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseManualTime(); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime(); +BENCHMARK(BM_MainThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_MainThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseRealTime(); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseManualTime(); +BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime(); +BENCHMARK(BM_MainThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_MainThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +// ========================================================================= // +// BM_WorkerThread + +void BM_WorkerThread(benchmark::State& state) { + for (auto _ : state) { + std::thread Worker(&MyBusySpinwait); + Worker.join(); + state.SetIterationTime(time_frame_in_sec); + } + state.counters["invtime"] = + benchmark::Counter{1, benchmark::Counter::kIsRate}; +} + +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseRealTime(); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseManualTime(); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime(); +BENCHMARK(BM_WorkerThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_WorkerThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseRealTime(); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseManualTime(); +BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime(); +BENCHMARK(BM_WorkerThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_WorkerThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +// ========================================================================= // +// BM_MainThreadAndWorkerThread + +void BM_MainThreadAndWorkerThread(benchmark::State& state) { + for (auto _ : state) { + std::thread Worker(&MyBusySpinwait); + MyBusySpinwait(); + Worker.join(); + state.SetIterationTime(time_frame_in_sec); + } + state.counters["invtime"] = + benchmark::Counter{1, benchmark::Counter::kIsRate}; +} + +BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(1); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(1) + ->UseRealTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(1) + ->UseManualTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(1) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(2); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(2) + ->UseRealTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(2) + ->UseManualTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseRealTime(); +BENCHMARK(BM_MainThreadAndWorkerThread) + ->Iterations(1) + ->Threads(2) + ->MeasureProcessCPUTime() + ->UseManualTime(); + +// ========================================================================= // +// ---------------------------- TEST CASES END ----------------------------- // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } From c5b2fe9357b3862b7f99b94d7999002dcf269faf Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 9 Apr 2019 13:02:45 +0100 Subject: [PATCH 138/330] Load http_archive for bazel build --- WORKSPACE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index 54734f1ea5..9a75f968d9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,7 @@ workspace(name = "com_github_google_benchmark") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( name = "com_google_googletest", urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], From 56fd56dc02a348f717461ffdaa8673a97bfbdb82 Mon Sep 17 00:00:00 2001 From: Jusufadis Bakamovic Date: Thu, 11 Apr 2019 11:48:29 +0200 Subject: [PATCH 139/330] Refactor U-Test calculation into separate function. (#740) * Refactor U-Test calculation into separate function. And implement 'print_utest' functionality in terms of it. * Change 'optimal_repetitions' to 'more_optimal_repetitions'. --- tools/gbench/report.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 49b46eca65..5bd3a8d85d 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -154,11 +154,7 @@ def extract_field(partition, field_name): rhs = [x[field_name] for x in partition[1]] return [lhs, rhs] - -def print_utest(partition, utest_alpha, first_col_width, use_color=True): - timings_time = extract_field(partition, 'real_time') - timings_cpu = extract_field(partition, 'cpu_time') - +def calc_utest(timings_cpu, timings_time): min_rep_cnt = min(len(timings_time[0]), len(timings_time[1]), len(timings_cpu[0]), @@ -166,21 +162,33 @@ def print_utest(partition, utest_alpha, first_col_width, use_color=True): # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions? if min_rep_cnt < UTEST_MIN_REPETITIONS: - return [] - - def get_utest_color(pval): - return BC_FAIL if pval >= utest_alpha else BC_OKGREEN + return False, None, None time_pvalue = mannwhitneyu( timings_time[0], timings_time[1], alternative='two-sided').pvalue cpu_pvalue = mannwhitneyu( timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue + return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue + +def print_utest(partition, utest_alpha, first_col_width, use_color=True): + def get_utest_color(pval): + return BC_FAIL if pval >= utest_alpha else BC_OKGREEN + + timings_time = extract_field(partition, 'real_time') + timings_cpu = extract_field(partition, 'cpu_time') + have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time) + + # Check if we failed miserably with minimum required repetitions for utest + if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None: + return [] + dsc = "U Test, Repetitions: {} vs {}".format( len(timings_cpu[0]), len(timings_cpu[1])) dsc_color = BC_OKGREEN - if min_rep_cnt < UTEST_OPTIMAL_REPETITIONS: + # We still got some results to show but issue a warning about it. + if not have_optimal_repetitions: dsc_color = BC_WARNING dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( UTEST_OPTIMAL_REPETITIONS) From 415835e03e5e78b5c17b450903c553a079214879 Mon Sep 17 00:00:00 2001 From: Hannes Hauswedell Date: Thu, 11 Apr 2019 15:36:11 +0000 Subject: [PATCH 140/330] fix master branch on *BSD (#792) * fix master branch on *BSD * add name to CONTRIBUTORS --- CONTRIBUTORS | 1 + src/sysinfo.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index c86d873564..bee57439ea 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -39,6 +39,7 @@ Eugene Zhuk Evgeny Safronov Federico Ficarelli Felix Homann +Hannes Hauswedell Ismael Jimenez Martinez Jern-Kuan Leong JianXiong Zhou diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 953c170cfd..01ecfad028 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -429,7 +429,7 @@ std::string GetSystemName() { #endif return str; #else // defined(BENCHMARK_OS_WINDOWS) -#ifdef BENCHMARK_OS_MACOSX //Mac Doesnt have HOST_NAME_MAX defined +#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_QNX) #define HOST_NAME_MAX 154 From 1d41de84633bf40ddf0ad50165d77200e3d8b5fd Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 17 Apr 2019 17:08:52 +0100 Subject: [PATCH 141/330] Add command line flags tests (#793) Increase coverage --- src/commandlineflags.cc | 4 ++ src/commandlineflags.h | 6 --- test/CMakeLists.txt | 1 + test/commandlineflags_gtest.cc | 78 ++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 test/commandlineflags_gtest.cc diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 734e88bbec..6bd65c5ae7 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -21,6 +21,8 @@ #include namespace benchmark { +namespace { + // Parses 'str' for a 32-bit signed integer. If successful, writes // the result to *value and returns true; otherwise leaves *value // unchanged and returns false. @@ -88,6 +90,8 @@ static std::string FlagToEnvVar(const char* flag) { return "BENCHMARK_" + env_var; } +} // namespace + // Reads and returns the Boolean environment variable corresponding to // the given flag; if it's not set, returns default_value. // diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 945c9a9fc4..5eaea82a59 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -23,16 +23,10 @@ std::string FLAG(name) = (default_val) namespace benchmark { -// Parses 'str' for a 32-bit signed integer. If successful, writes the result -// to *value and returns true; otherwise leaves *value unchanged and returns -// false. -bool ParseInt32(const std::string& src_text, const char* str, int32_t* value); - // Parses a bool/Int32/string from the environment variable // corresponding to the given Google Test flag. bool BoolFromEnv(const char* flag, bool default_val); int32_t Int32FromEnv(const char* flag, int32_t default_val); -double DoubleFromEnv(const char* flag, double default_val); const char* StringFromEnv(const char* flag, const char* default_val); // Parses a string for a bool flag, in the form of either diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 62370a54eb..d3a07639bf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -195,6 +195,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(benchmark_gtest) add_gtest(benchmark_name_gtest) + add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc new file mode 100644 index 0000000000..5460778c48 --- /dev/null +++ b/test/commandlineflags_gtest.cc @@ -0,0 +1,78 @@ +#include + +#include "../src/commandlineflags.h" +#include "../src/internal_macros.h" +#include "gtest/gtest.h" + +namespace benchmark { +namespace { + +#if defined(BENCHMARK_OS_WINDOWS) +int setenv(const char* name, const char* value, int overwrite) { + if (!overwrite) { + // NOTE: getenv_s is far superior but not available under mingw. + char* env_value = getenv(name); + if (env_value == nullptr) { + return -1; + } + } + return _putenv_s(name, value); +} + +int unsetenv(const char* name) { + return _putenv_s(name, ""); +} + +#endif // BENCHMARK_OS_WINDOWS + +TEST(BoolFromEnv, Default) { + ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + EXPECT_EQ(BoolFromEnv("not_in_env", true), true); +} + +TEST(BoolFromEnv, False) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); +} + +TEST(BoolFromEnv, True) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "1", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); +} + +TEST(Int32FromEnv, NotInEnv) { + ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + EXPECT_EQ(Int32FromEnv("not_in_env", 42), 42); +} + +TEST(Int32FromEnv, InvalidInteger) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + EXPECT_EQ(Int32FromEnv("in_env", 42), 42); + ASSERT_EQ(unsetenv("BENCHMARK_IN_ENV"), 0); +} + +TEST(Int32FromEnv, ValidInteger) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "42", 1), 0); + EXPECT_EQ(Int32FromEnv("in_env", 64), 42); + unsetenv("BENCHMARK_IN_ENV"); +} + +TEST(StringFromEnv, Default) { + ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + EXPECT_STREQ(StringFromEnv("not_in_env", "foo"), "foo"); +} + +TEST(StringFromEnv, Valid) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + EXPECT_STREQ(StringFromEnv("in_env", "bar"), "foo"); + unsetenv("BENCHMARK_IN_ENV"); +} + +} // namespace +} // namespace benchmark From 588be0446a9cc96315ffde428dd2a4fecfb8c44d Mon Sep 17 00:00:00 2001 From: Michael Tesch Date: Fri, 19 Apr 2019 19:47:25 +0200 Subject: [PATCH 142/330] escape special chars in csv and json output. (#802) * escape special chars in csv and json output. - escape \b,\f,\n,\r,\t,\," from strings before dumping them to json or csv. - also faithfully reproduce the sign of nan in json. this fixes github issue #745. * functionalize. * split string escape functions between csv and json * Update src/csv_reporter.cc Co-Authored-By: tesch1 * Update src/json_reporter.cc Co-Authored-By: tesch1 --- src/csv_reporter.cc | 29 ++++++++++++++-------------- src/json_reporter.cc | 37 ++++++++++++++++++++++++------------ src/string_util.cc | 9 --------- src/string_util.h | 3 --- test/reporter_output_test.cc | 31 ++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 38 deletions(-) diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index d2f1d27eb6..af2c18fc8a 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -37,6 +37,18 @@ std::vector elements = { "error_occurred", "error_message"}; } // namespace +std::string CsvEscape(const std::string & s) { + std::string tmp; + tmp.reserve(s.size() + 2); + for (char c : s) { + switch (c) { + case '"' : tmp += "\"\""; break; + default : tmp += c; break; + } + } + return '"' + tmp + '"'; +} + bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; @@ -89,18 +101,11 @@ void CSVReporter::ReportRuns(const std::vector& reports) { void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); - - // Field with embedded double-quote characters must be doubled and the field - // delimited with double-quotes. - std::string name = run.benchmark_name(); - ReplaceAll(&name, "\"", "\"\""); - Out << '"' << name << "\","; + Out << CsvEscape(run.benchmark_name()) << ","; if (run.error_occurred) { Out << std::string(elements.size() - 3, ','); Out << "true,"; - std::string msg = run.error_message; - ReplaceAll(&msg, "\"", "\"\""); - Out << '"' << msg << "\"\n"; + Out << CsvEscape(run.error_message) << "\n"; return; } @@ -130,11 +135,7 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; if (!run.report_label.empty()) { - // Field with embedded double-quote characters must be doubled and the field - // delimited with double-quotes. - std::string label = run.report_label; - ReplaceAll(&label, "\"", "\"\""); - Out << "\"" << label << "\""; + Out << CsvEscape(run.report_label); } Out << ",,"; // for error_occurred and error_message diff --git a/src/json_reporter.cc b/src/json_reporter.cc index dd4ba30e38..cf1de2547a 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -32,30 +32,48 @@ namespace benchmark { namespace { +std::string StrEscape(const std::string & s) { + std::string tmp; + tmp.reserve(s.size()); + for (char c : s) { + switch (c) { + case '\b': tmp += "\\b"; break; + case '\f': tmp += "\\f"; break; + case '\n': tmp += "\\n"; break; + case '\r': tmp += "\\r"; break; + case '\t': tmp += "\\t"; break; + case '\\': tmp += "\\\\"; break; + case '"' : tmp += "\\\""; break; + default : tmp += c; break; + } + } + return tmp; +} + std::string FormatKV(std::string const& key, std::string const& value) { - return StrFormat("\"%s\": \"%s\"", key.c_str(), value.c_str()); + return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); } std::string FormatKV(std::string const& key, const char* value) { - return StrFormat("\"%s\": \"%s\"", key.c_str(), value); + return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); } std::string FormatKV(std::string const& key, bool value) { - return StrFormat("\"%s\": %s", key.c_str(), value ? "true" : "false"); + return StrFormat("\"%s\": %s", StrEscape(key).c_str(), value ? "true" : "false"); } std::string FormatKV(std::string const& key, int64_t value) { std::stringstream ss; - ss << '"' << key << "\": " << value; + ss << '"' << StrEscape(key) << "\": " << value; return ss.str(); } std::string FormatKV(std::string const& key, double value) { std::stringstream ss; - ss << '"' << key << "\": "; + ss << '"' << StrEscape(key) << "\": "; if (std::isnan(value)) - ss << "NaN"; + ss << (value < 0 ? "-" : "") << "NaN"; else if (std::isinf(value)) ss << (value < 0 ? "-" : "") << "Infinity"; else { @@ -88,12 +106,7 @@ bool JSONReporter::ReportContext(const Context& context) { out << indent << FormatKV("host_name", context.sys_info.name) << ",\n"; if (Context::executable_name) { - // windows uses backslash for its path separator, - // which must be escaped in JSON otherwise it blows up conforming JSON - // decoders - std::string executable_name = Context::executable_name; - ReplaceAll(&executable_name, "\\", "\\\\"); - out << indent << FormatKV("executable", executable_name) << ",\n"; + out << indent << FormatKV("executable", Context::executable_name) << ",\n"; } CPUInfo const& info = context.cpu_info; diff --git a/src/string_util.cc b/src/string_util.cc index 05ac5b4ea3..39b01a1719 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -160,15 +160,6 @@ std::string StrFormat(const char* format, ...) { return tmp; } -void ReplaceAll(std::string* str, const std::string& from, - const std::string& to) { - std::size_t start = 0; - while ((start = str->find(from, start)) != std::string::npos) { - str->replace(start, from.length(), to); - start += to.length(); - } -} - #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/src/string_util.h b/src/string_util.h index c5dcee6fcd..09d7b4bd2a 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -37,9 +37,6 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } -void ReplaceAll(std::string* str, const std::string& from, - const std::string& to); - #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8b7ae935aa..c8090d4aca 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -704,6 +704,37 @@ ADD_CASES( "manual_time_stddev\",%csv_report$"}, {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); +// ========================================================================= // +// ------------------------- Testing StrEscape JSON ------------------------ // +// ========================================================================= // +#if 0 // enable when csv testing code correctly handles multi-line fields +void BM_JSON_Format(benchmark::State& state) { + state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes"); + for (auto _ : state) { + } +} +BENCHMARK(BM_JSON_Format); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, + {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"error_occurred\": true,$", MR_Next}, + {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}}); +#endif +// ========================================================================= // +// -------------------------- Testing CsvEscape ---------------------------- // +// ========================================================================= // + +void BM_CSV_Format(benchmark::State& state) { + state.SkipWithError("\"freedom\""); + for (auto _ : state) { + } +} +BENCHMARK(BM_CSV_Format); +ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}}); + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // From 05d8c1c5d09028a39306c368791c0aa545ad19e7 Mon Sep 17 00:00:00 2001 From: astee Date: Thu, 25 Apr 2019 04:48:56 -0700 Subject: [PATCH 143/330] Improve README (#804) * Update and organize README * Update AUTHORS and CONTRIBUTORS Fixes #803. --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 1062 ++++++++++++++++++++++++++++---------------------- 3 files changed, 597 insertions(+), 467 deletions(-) diff --git a/AUTHORS b/AUTHORS index d5cd007895..912cbbc13c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -9,6 +9,7 @@ # Please keep the list sorted. Albert Pretorius +Alex Steele Andriy Berestovskyy Arne Beer Carto diff --git a/CONTRIBUTORS b/CONTRIBUTORS index bee57439ea..b680efc8c4 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -23,6 +23,7 @@ # Please keep the list sorted. Albert Pretorius +Alex Steele Andriy Berestovskyy Arne Beer Billy Robert O'Neal III diff --git a/README.md b/README.md index 11d7a8b347..251c5cd673 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,36 @@ -# benchmark +# Benchmark [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) [![slackin](https://slackin-iqtfqnpzxd.now.sh/badge.svg)](https://slackin-iqtfqnpzxd.now.sh/) -A library to support the benchmarking of functions, similar to unit-tests. + +A library to benchmark code snippets, similar to unit tests. Example: + +```c++ +#include + +static void BM_SomeFunction(benchmark::State& state) { + // Perform setup here + for (auto _ : state) { + // This code gets timed + SomeFunction(); + } +} +// Register the function as a benchmark +BENCHMARK(BM_SomeFunction); +// Run the benchmark +BENCHMARK_MAIN(); +``` + +To get started, see [Requirements](#requirements) and +[Installation](#installation). See [Usage](#usage) for a full example and the +[User Guide](#user-guide) for a more comprehensive feature overview. + +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) +as some of the structural aspects of the APIs are similar. + +### Resources [Discussion group](https://groups.google.com/d/forum/benchmark-discuss) @@ -14,20 +40,60 @@ IRC channel: [freenode](https://freenode.net) #googlebenchmark [Assembly Testing Documentation](docs/AssemblyTests.md) +## Requirements + +The library can be used with C++03. However, it requires C++11 to build, +including compiler and standard library support. -## Building +The following minimum versions are required to build the library: -The basic steps for configuring and building the library look like this: +* GCC 4.8 +* Clang 3.4 +* Visual Studio 2013 +* Intel 2015 Update 1 + +## Installation + +This describes the installation process using cmake. As pre-requisites, you'll +need git and cmake installed. ```bash +# Check out the library. $ git clone https://github.com/google/benchmark.git # Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. $ git clone https://github.com/google/googletest.git benchmark/googletest +# Make a build directory to place the build output. $ mkdir build && cd build -$ cmake -G [options] ../benchmark -# Assuming a makefile generator was used +# Generate a Makefile with cmake. +# Use cmake -G to generate a different file type. +$ cmake ../benchmark +# Build the library. $ make ``` +This builds the `benchmark` and `benchmark_main` libraries and tests. +On a unix system, the build directory should now look something like this: + +``` +/benchmark +/build + /src + /libbenchmark.a + /libbenchmark_main.a + /test + ... +``` + +Next, you can run the tests to check the build. + +```bash +$ make test +``` + +If you want to install the library globally, also run: + +``` +sudo make install +``` Note that Google Benchmark requires Google Test to build and run the tests. This dependency can be provided two ways: @@ -40,37 +106,29 @@ dependency can be provided two ways: If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF` to `CMAKE_ARGS`. +### Debug vs Release -## Installation Guide - -For Ubuntu and Debian Based System - -First make sure you have git and cmake installed (If not please install them) +By default, benchmark builds as a debug library. You will see a warning in the +output when this is the case. To build it as a release library instead, use: ``` -sudo apt-get install git cmake +cmake -DCMAKE_BUILD_TYPE=Release ``` -Now, let's clone the repository and build it +To enable link-time optimisation, use ``` -git clone https://github.com/google/benchmark.git -cd benchmark -# If you want to build tests and don't use BENCHMARK_DOWNLOAD_DEPENDENCIES, then -# git clone https://github.com/google/googletest.git -mkdir build -cd build -cmake .. -DCMAKE_BUILD_TYPE=RELEASE -make +cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true ``` -If you need to install the library globally +If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake +cache variables, if autodetection fails. -``` -sudo make install -``` +If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, +`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. -## Stable and Experimental Library Versions + +### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; the API of which can be considered largely stable, with source breaking changes @@ -82,16 +140,11 @@ to use, test, and provide feedback on the new features are encouraged to try this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. -## Further knowledge - -It may help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) -as some of the structural aspects of the APIs are similar. - -## Example usage +## Usage ### Basic usage -Define a function that executes the code to be measured, register it as a -benchmark function using the `BENCHMARK` macro, and ensure an appropriate `main` -function is available: +Define a function that executes the code to measure, register it as a benchmark +function using the `BENCHMARK` macro, and ensure an appropriate `main` function +is available: ```c++ #include @@ -114,15 +167,25 @@ BENCHMARK(BM_StringCopy); BENCHMARK_MAIN(); ``` -Don't forget to inform your linker to add benchmark library e.g. through -`-lbenchmark` compilation flag. Alternatively, you may leave out the -`BENCHMARK_MAIN();` at the end of the source file and link against -`-lbenchmark_main` to get the same default behavior. +To run the benchmark, compile and link against the `benchmark` library +(libbenchmark.a/.so). If you followed the build steps above, this +library will be under the build directory you created. + +```bash +# Example on linux after running the build steps above. Assumes the +# `benchmark` and `build` directories are under the current directory. +$ g++ -std=c++11 -isystem benchmark/include -Lbuild/src -lpthread \ + -lbenchmark mybenchmark.cc -o mybenchmark +``` + +Alternatively, link against the `benchmark_main` library and remove +`BENCHMARK_MAIN();` above to get the same behavior. -The benchmark library will measure and report the timing for code within the -`for(...)` loop. +The compiled executable will run all benchmarks by default. Pass the `--help` +flag for option information or see the guide below. + +### Platform-specific instructions -#### Platform-specific libraries When the library is built using GCC it is necessary to link with the pthread library due to how GCC implements `std::thread`. Failing to link to pthread will lead to runtime exceptions (unless you're using libc++), not linker errors. See @@ -137,7 +200,185 @@ also required. If you're running benchmarks on solaris, you'll want the kstat library linked in too (`-lkstat`). -### Passing arguments +## User Guide + +### Command Line +[Output Formats](#output-formats) + +[Output Files](#output-files) + +[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks) + +[Result Comparison](#result-comparison) + +### Library +[Runtime and Reporting Considerations](#runtime-and-reporting-considerations) + +[Passing Arguments](#passing-arguments) + +[Calculating Asymptotic Complexity](#asymptotic-complexity) + +[Templated Benchmarks](#templated-benchmarks) + +[Fixtures](#fixtures) + +[Custom Counters](#custom-counters) + +[Multithreaded Benchmarks](#multithreaded-benchmarks) + +[CPU Timers](#cpu-timers) + +[Manual Timing](#manual-timing) + +[Setting the Time Unit](#setting-the-time-unit) + +[Preventing Optimization](#preventing-optimization) + +[Reporting Statistics](#reporting-statistics) + +[Custom Statistics](#custom-statistics) + +[Using RegisterBenchmark](#using-register-benchmark) + +[Exiting with an Error](#exiting-with-an-error) + +[A Faster KeepRunning Loop](#a-faster-keep-running-loop) + +[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) + + + +### Output Formats + +The library supports multiple output formats. Use the +`--benchmark_format=` flag to set the format type. `console` +is the default format. + +The Console format is intended to be a human readable format. By default +the format generates color output. Context is output on stderr and the +tabular data on stdout. Example tabular output looks like: +``` +Benchmark Time(ns) CPU(ns) Iterations +---------------------------------------------------------------------- +BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s +BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s +BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s +``` + +The JSON format outputs human readable json split into two top level attributes. +The `context` attribute contains information about the run in general, including +information about the CPU and the date. +The `benchmarks` attribute contains a list of every benchmark run. Example json +output looks like: +```json +{ + "context": { + "date": "2015/03/17-18:40:25", + "num_cpus": 40, + "mhz_per_cpu": 2801, + "cpu_scaling_enabled": false, + "build_type": "debug" + }, + "benchmarks": [ + { + "name": "BM_SetInsert/1024/1", + "iterations": 94877, + "real_time": 29275, + "cpu_time": 29836, + "bytes_per_second": 134066, + "items_per_second": 33516 + }, + { + "name": "BM_SetInsert/1024/8", + "iterations": 21609, + "real_time": 32317, + "cpu_time": 32429, + "bytes_per_second": 986770, + "items_per_second": 246693 + }, + { + "name": "BM_SetInsert/1024/10", + "iterations": 21393, + "real_time": 32724, + "cpu_time": 33355, + "bytes_per_second": 1199226, + "items_per_second": 299807 + } + ] +} +``` + +The CSV format outputs comma-separated values. The `context` is output on stderr +and the CSV itself on stdout. Example CSV output looks like: +``` +name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label +"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, +"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, +"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, +``` + + + +### Output Files + +Write benchmark results to a file with the `--benchmark_out=` option. +Specify the output format with `--benchmark_out_format={json|console|csv}`. Note that Specifying +`--benchmark_out` does not suppress the console output. + + + +### Running a Subset of Benchmarks + +The `--benchmark_filter=` option can be used to only run the benchmarks +which match the specified ``. For example: + +```bash +$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 +Run on (1 X 2300 MHz CPU ) +2016-06-25 19:34:24 +Benchmark Time CPU Iterations +---------------------------------------------------- +BM_memcpy/32 11 ns 11 ns 79545455 +BM_memcpy/32k 2181 ns 2185 ns 324074 +BM_memcpy/32 12 ns 12 ns 54687500 +BM_memcpy/32k 1834 ns 1837 ns 357143 +``` + + + +### Result comparison + +It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) + + + +### Runtime and Reporting Considerations + +When the benchmark binary is executed, each benchmark function is run serially. +The number of iterations to run is determined dynamically by running the +benchmark a few times and measuring the time taken and ensuring that the +ultimate result will be statistically stable. As such, faster benchmark +functions will be run for more iterations than slower benchmark functions, and +the number of iterations is thus reported. + +In all cases, the number of iterations for which the benchmark is run is +governed by the amount of time the benchmark takes. Concretely, the number of +iterations is at least one, not more than 1e9, until CPU time is greater than +the minimum time, or the wallclock time is 5x minimum time. The minimum time is +set per benchmark by calling `MinTime` on the registered benchmark object. + +Average timings are then reported over the iterations run. If multiple +repetitions are requested using the `--benchmark_repetitions` command-line +option, or at registration time, the benchmark function will be run several +times and statistical results across these repetitions will also be reported. + +As well as the per-benchmark entries, a preamble in the report will include +information about the machine on which the benchmarks are run. + + + +### Passing Arguments + Sometimes a family of benchmarks can be implemented with just one routine that takes an extra argument to specify which one of the family of benchmarks to run. For example, the following code defines a family of benchmarks for @@ -224,7 +465,31 @@ static void CustomArguments(benchmark::internal::Benchmark* b) { BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` -### Calculate asymptotic complexity (Big O) +#### Passing Arbitrary Arguments to a Benchmark + +In C++11 it is possible to define a benchmark that takes an arbitrary number +of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` +macro creates a benchmark that invokes `func` with the `benchmark::State` as +the first argument followed by the specified `args...`. +The `test_case_name` is appended to the name of the benchmark and +should describe the values passed. + +```c++ +template +void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { + [...] +} +// Registers a benchmark named "BM_takes_args/int_string_test" that passes +// the specified values to `extra_args`. +BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +``` +Note that elements of `...args` may refer to global variables. Users should +avoid modifying global state inside of a benchmark. + + + +### Calculating Asymptotic Complexity (Big O) + Asymptotic complexity might be calculated for a family of benchmarks. The following code will calculate the coefficient for the high-order term in the running time and the normalized root-mean square error of string comparison. @@ -258,10 +523,12 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; }); ``` -### Templated benchmarks -Templated benchmarks work the same way: This example produces and consumes -messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the -absence of multiprogramming. + + +### Templated Benchmarks + +This example produces and consumes messages of size `sizeof(v)` `range_x` +times. It also outputs throughput in the absence of multiprogramming. ```c++ template void BM_Sequential(benchmark::State& state) { @@ -292,110 +559,210 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` -### A Faster KeepRunning loop + -In C++11 mode, a ranged-based for loop should be used in preference to -the `KeepRunning` loop for running the benchmarks. For example: +### Fixtures + +Fixture tests are created by first defining a type that derives from +`::benchmark::Fixture` and then creating/registering the tests using the +following macros: + +* `BENCHMARK_F(ClassName, Method)` +* `BENCHMARK_DEFINE_F(ClassName, Method)` +* `BENCHMARK_REGISTER_F(ClassName, Method)` + +For Example: ```c++ -static void BM_Fast(benchmark::State &state) { - for (auto _ : state) { - FastOperation(); +class MyFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) { } -} -BENCHMARK(BM_Fast); -``` -The reason the ranged-for loop is faster than using `KeepRunning`, is -because `KeepRunning` requires a memory load and store of the iteration count -ever iteration, whereas the ranged-for variant is able to keep the iteration count -in a register. + void TearDown(const ::benchmark::State& state) { + } +}; -For example, an empty inner loop of using the ranged-based for method looks like: +BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} -```asm -# Loop Init - mov rbx, qword ptr [r14 + 104] - call benchmark::State::StartKeepRunning() - test rbx, rbx - je .LoopEnd -.LoopHeader: # =>This Inner Loop Header: Depth=1 - add rbx, -1 - jne .LoopHeader -.LoopEnd: +BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} +/* BarTest is NOT registered */ +BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); +/* BarTest is now registered */ ``` -Compared to an empty `KeepRunning` loop, which looks like: +#### Templated Fixtures -```asm -.LoopHeader: # in Loop: Header=BB0_3 Depth=1 - cmp byte ptr [rbx], 1 - jne .LoopInit -.LoopBody: # =>This Inner Loop Header: Depth=1 - mov rax, qword ptr [rbx + 8] - lea rcx, [rax + 1] - mov qword ptr [rbx + 8], rcx - cmp rax, qword ptr [rbx + 104] - jb .LoopHeader - jmp .LoopEnd -.LoopInit: - mov rdi, rbx - call benchmark::State::StartKeepRunning() - jmp .LoopBody -.LoopEnd: +Also you can create templated fixture by using the following macros: + +* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)` +* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` + +For example: +```c++ +template +class MyFixture : public benchmark::Fixture {}; + +BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} + +BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) { + for (auto _ : st) { + ... + } +} + +BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); ``` -Unless C++03 compatibility is required, the ranged-for variant of writing -the benchmark loop should be preferred. + -## Passing arbitrary arguments to a benchmark -In C++11 it is possible to define a benchmark that takes an arbitrary number -of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` -macro creates a benchmark that invokes `func` with the `benchmark::State` as -the first argument followed by the specified `args...`. -The `test_case_name` is appended to the name of the benchmark and -should describe the values passed. +### Custom Counters + +You can add your own counters with user-defined names. The example below +will add columns "Foo", "Bar" and "Baz" in its output: ```c++ -template -void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { - [...] +static void UserCountersExample1(benchmark::State& state) { + double numFoos = 0, numBars = 0, numBazs = 0; + for (auto _ : state) { + // ... count Foo,Bar,Baz events + } + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; } -// Registers a benchmark named "BM_takes_args/int_string_test" that passes -// the specified values to `extra_args`. -BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); ``` -Note that elements of `...args` may refer to global variables. Users should -avoid modifying global state inside of a benchmark. -## Using RegisterBenchmark(name, fn, args...) +The `state.counters` object is a `std::map` with `std::string` keys +and `Counter` values. The latter is a `double`-like class, via an implicit +conversion to `double&`. Thus you can use all of the standard arithmetic +assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. -The `RegisterBenchmark(name, func, args...)` function provides an alternative -way to create and register benchmarks. -`RegisterBenchmark(name, func, args...)` creates, registers, and returns a -pointer to a new benchmark with the specified `name` that invokes -`func(st, args...)` where `st` is a `benchmark::State` object. +In multithreaded benchmarks, each counter is set on the calling thread only. +When the benchmark finishes, the counters from each thread will be summed; +the resulting sum is the value which will be shown for the benchmark. -Unlike the `BENCHMARK` registration macros, which can only be used at the global -scope, the `RegisterBenchmark` can be called anywhere. This allows for -benchmark tests to be registered programmatically. +The `Counter` constructor accepts three parameters: the value as a `double` +; a bit flag which allows you to show counters as rates, and/or as per-thread +iteration, and/or as per-thread averages, and/or iteration invariants; +and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, +`benchmark::Counter::OneK::kIs1000`), or 1024 +(`benchmark::Counter::OneK::kIs1024`)? -Additionally `RegisterBenchmark` allows any callable object to be registered -as a benchmark. Including capturing lambdas and function objects. +```c++ + // sets a simple counter + state.counters["Foo"] = numFoos; + + // Set the counter as a rate. It will be presented divided + // by the duration of the benchmark. + state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); + + // Set the counter as a thread-average quantity. It will + // be presented divided by the number of threads. + state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); + + // There's also a combined flag: + state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); + + // This says that we process with the rate of state.range(0) bytes every iteration: + state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); +``` + +When you're compiling in C++11 mode or later you can use `insert()` with +`std::initializer_list`: -For Example: ```c++ -auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; + // With C++11, this can be done: + state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); + // ... instead of: + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; +``` -int main(int argc, char** argv) { - for (auto& test_input : { /* ... */ }) - benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); -} +#### Counter Reporting + +When using the console reporter, by default, user counters are are printed at +the end after the table, the same way as ``bytes_processed`` and +``items_processed``. This is best for cases in which there are few counters, +or where there are only a couple of lines per benchmark. Here's an example of +the default output: + +``` +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations UserCounters... +------------------------------------------------------------------------------ +BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m +BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2 +BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4 +BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16 +BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32 +BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4 +BM_Factorial 26 ns 26 ns 26608979 40320 +BM_Factorial/real_time 26 ns 26 ns 26587936 40320 +BM_CalculatePiRange/1 16 ns 16 ns 45704255 0 +BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374 +BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746 +BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355 +``` + +If this doesn't suit you, you can print each counter as a table column by +passing the flag `--benchmark_counters_tabular=true` to the benchmark +application. This is best for cases in which there are a lot of counters, or +a lot of lines per individual benchmark. Note that this will trigger a +reprinting of the table header any time the counter set changes between +individual benchmarks. Here's an example of corresponding output when +`--benchmark_counters_tabular=true` is passed: + +``` +--------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations Bar Bat Baz Foo +--------------------------------------------------------------------------------------- +BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8 +BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1 +BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2 +BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4 +BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8 +BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16 +BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32 +BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4 +-------------------------------------------------------------- +Benchmark Time CPU Iterations +-------------------------------------------------------------- +BM_Factorial 26 ns 26 ns 26392245 40320 +BM_Factorial/real_time 26 ns 26 ns 26494107 40320 +BM_CalculatePiRange/1 15 ns 15 ns 45571597 0 +BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374 +BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746 +BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355 +BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184 +BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162 +BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 +BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 +BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 ``` +Note above the additional header printed when the benchmark changes from +``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does +not have the same counter set as ``BM_UserCounter``. + + + +### Multithreaded Benchmarks -### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have reached the start of the benchmark loop, and all will have finished before any thread @@ -428,7 +795,10 @@ BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); Without `UseRealTime`, CPU time is used by default. -## CPU timers + + +### CPU Timers + By default, the CPU timer only measures the time spent by the main thread. If the benchmark itself uses threads internally, this measurement may not be what you are looking for. Instead, there is a way to measure the total @@ -471,9 +841,10 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime(); BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime(); ``` -## Controlling timers +#### Controlling Timers + Normally, the entire duration of the work loop (`for (auto _ : state) {}`) -is measured. But sometimes, it is nessesary to do some work inside of +is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. That is possible, althought it is not recommended, since it has high overhead. @@ -492,7 +863,10 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -## Manual timing + + +### Manual Timing + For benchmarking something for which neither CPU time nor real-time are correct or accurate enough, completely manual timing is supported using the `UseManualTime` function. @@ -530,7 +904,22 @@ static void BM_ManualTiming(benchmark::State& state) { BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); ``` -### Preventing optimisation + + +### Setting the Time Unit + +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +```c++ +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +``` + + + +### Preventing Optimization + To prevent a value or expression from being optimized away by the compiler the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()` functions can be used. @@ -588,16 +977,10 @@ static void BM_vector_push_back(benchmark::State& state) { Note that `ClobberMemory()` is only available for GNU or MSVC based compilers. -### Set time unit manually -If a benchmark runs a few milliseconds it may be hard to visually compare the -measured times, since the output data is given in nanoseconds per default. In -order to manually set the time unit, you can specify it manually: + -```c++ -BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); -``` +### Statistics: Reporting the Mean, Median and Standard Deviation of Repeated Benchmarks -### Reporting the mean, median and standard deviation by repeated benchmarks By default each benchmark is run once and that single result is reported. However benchmarks are often noisy and a single result may not be representative of the overall behavior. For this reason it's possible to repeatedly rerun the @@ -623,10 +1006,13 @@ Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a registered benchmark object overrides the value of the appropriate flag for that benchmark. -## User-defined statistics for repeated benchmarks + + +### Custom Statistics + While having mean, median and standard deviation is nice, this may not be -enough for everyone. For example you may want to know what is the largest -observation, e.g. because you have some real-time constraints. This is easy. +enough for everyone. For example you may want to know what the largest +observation is, e.g. because you have some real-time constraints. This is easy. The following code will specify a custom statistic to be calculated, defined by a lambda function. @@ -646,201 +1032,38 @@ BENCHMARK(BM_spin_empty) ->Arg(512); ``` -## Fixtures -Fixture tests are created by -first defining a type that derives from `::benchmark::Fixture` and then -creating/registering the tests using the following macros: + -* `BENCHMARK_F(ClassName, Method)` -* `BENCHMARK_DEFINE_F(ClassName, Method)` -* `BENCHMARK_REGISTER_F(ClassName, Method)` +### Using RegisterBenchmark(name, fn, args...) -For Example: +The `RegisterBenchmark(name, func, args...)` function provides an alternative +way to create and register benchmarks. +`RegisterBenchmark(name, func, args...)` creates, registers, and returns a +pointer to a new benchmark with the specified `name` that invokes +`func(st, args...)` where `st` is a `benchmark::State` object. -```c++ -class MyFixture : public benchmark::Fixture { -public: - void SetUp(const ::benchmark::State& state) { - } +Unlike the `BENCHMARK` registration macros, which can only be used at the global +scope, the `RegisterBenchmark` can be called anywhere. This allows for +benchmark tests to be registered programmatically. - void TearDown(const ::benchmark::State& state) { - } -}; +Additionally `RegisterBenchmark` allows any callable object to be registered +as a benchmark. Including capturing lambdas and function objects. -BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} - -BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} -/* BarTest is NOT registered */ -BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); -/* BarTest is now registered */ -``` - -### Templated fixtures -Also you can create templated fixture by using the following macros: - -* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)` -* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` - -For example: +For Example: ```c++ -template -class MyFixture : public benchmark::Fixture {}; - -BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} - -BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) { - for (auto _ : st) { - ... - } -} - -BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2); -``` - -## User-defined counters - -You can add your own counters with user-defined names. The example below -will add columns "Foo", "Bar" and "Baz" in its output: +auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; -```c++ -static void UserCountersExample1(benchmark::State& state) { - double numFoos = 0, numBars = 0, numBazs = 0; - for (auto _ : state) { - // ... count Foo,Bar,Baz events - } - state.counters["Foo"] = numFoos; - state.counters["Bar"] = numBars; - state.counters["Baz"] = numBazs; +int main(int argc, char** argv) { + for (auto& test_input : { /* ... */ }) + benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); } ``` -The `state.counters` object is a `std::map` with `std::string` keys -and `Counter` values. The latter is a `double`-like class, via an implicit -conversion to `double&`. Thus you can use all of the standard arithmetic -assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. - -In multithreaded benchmarks, each counter is set on the calling thread only. -When the benchmark finishes, the counters from each thread will be summed; -the resulting sum is the value which will be shown for the benchmark. - -The `Counter` constructor accepts three parameters: the value as a `double` -; a bit flag which allows you to show counters as rates, and/or as per-thread -iteration, and/or as per-thread averages, and/or iteration invariants; -and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, -`benchmark::Counter::OneK::kIs1000`), or 1024 -(`benchmark::Counter::OneK::kIs1024`)? - -```c++ - // sets a simple counter - state.counters["Foo"] = numFoos; - - // Set the counter as a rate. It will be presented divided - // by the duration of the benchmark. - state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); - - // Set the counter as a thread-average quantity. It will - // be presented divided by the number of threads. - state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); + - // There's also a combined flag: - state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); - - // This says that we process with the rate of state.range(0) bytes every iteration: - state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024); -``` - -When you're compiling in C++11 mode or later you can use `insert()` with -`std::initializer_list`: - -```c++ - // With C++11, this can be done: - state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); - // ... instead of: - state.counters["Foo"] = numFoos; - state.counters["Bar"] = numBars; - state.counters["Baz"] = numBazs; -``` - -### Counter reporting - -When using the console reporter, by default, user counters are are printed at -the end after the table, the same way as ``bytes_processed`` and -``items_processed``. This is best for cases in which there are few counters, -or where there are only a couple of lines per benchmark. Here's an example of -the default output: - -``` ------------------------------------------------------------------------------- -Benchmark Time CPU Iterations UserCounters... ------------------------------------------------------------------------------- -BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8 -BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m -BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2 -BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4 -BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8 -BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16 -BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32 -BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4 -BM_Factorial 26 ns 26 ns 26608979 40320 -BM_Factorial/real_time 26 ns 26 ns 26587936 40320 -BM_CalculatePiRange/1 16 ns 16 ns 45704255 0 -BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374 -BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746 -BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355 -``` - -If this doesn't suit you, you can print each counter as a table column by -passing the flag `--benchmark_counters_tabular=true` to the benchmark -application. This is best for cases in which there are a lot of counters, or -a lot of lines per individual benchmark. Note that this will trigger a -reprinting of the table header any time the counter set changes between -individual benchmarks. Here's an example of corresponding output when -`--benchmark_counters_tabular=true` is passed: - -``` ---------------------------------------------------------------------------------------- -Benchmark Time CPU Iterations Bar Bat Baz Foo ---------------------------------------------------------------------------------------- -BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8 -BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1 -BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2 -BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4 -BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8 -BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16 -BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32 -BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4 --------------------------------------------------------------- -Benchmark Time CPU Iterations --------------------------------------------------------------- -BM_Factorial 26 ns 26 ns 26392245 40320 -BM_Factorial/real_time 26 ns 26 ns 26494107 40320 -BM_CalculatePiRange/1 15 ns 15 ns 45571597 0 -BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374 -BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746 -BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355 -BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184 -BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162 -BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 -BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 -BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 -``` -Note above the additional header printed when the benchmark changes from -``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does -not have the same counter set as ``BM_UserCounter``. - -## Exiting Benchmarks in Error +### Exiting with an Error When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the @@ -880,162 +1103,67 @@ static void BM_test_ranged_fo(benchmark::State & state) { } } ``` + -## Running a subset of the benchmarks - -The `--benchmark_filter=` option can be used to only run the benchmarks -which match the specified ``. For example: - -```bash -$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 -Run on (1 X 2300 MHz CPU ) -2016-06-25 19:34:24 -Benchmark Time CPU Iterations ----------------------------------------------------- -BM_memcpy/32 11 ns 11 ns 79545455 -BM_memcpy/32k 2181 ns 2185 ns 324074 -BM_memcpy/32 12 ns 12 ns 54687500 -BM_memcpy/32k 1834 ns 1837 ns 357143 -``` - -## Runtime and reporting considerations -When the benchmark binary is executed, each benchmark function is run serially. -The number of iterations to run is determined dynamically by running the -benchmark a few times and measuring the time taken and ensuring that the -ultimate result will be statistically stable. As such, faster benchmark -functions will be run for more iterations than slower benchmark functions, and -the number of iterations is thus reported. - -In all cases, the number of iterations for which the benchmark is run is -governed by the amount of time the benchmark takes. Concretely, the number of -iterations is at least one, not more than 1e9, until CPU time is greater than -the minimum time, or the wallclock time is 5x minimum time. The minimum time is -set per benchmark by calling `MinTime` on the registered benchmark object. - -Average timings are then reported over the iterations run. If multiple -repetitions are requested using the `--benchmark_repetitions` command-line -option, or at registration time, the benchmark function will be run several -times and statistical results across these repetitions will also be reported. - -As well as the per-benchmark entries, a preamble in the report will include -information about the machine on which the benchmarks are run. - -### Output Formats -The library supports multiple output formats. Use the -`--benchmark_format=` flag to set the format type. `console` -is the default format. +### A Faster KeepRunning Loop -The Console format is intended to be a human readable format. By default -the format generates color output. Context is output on stderr and the -tabular data on stdout. Example tabular output looks like: -``` -Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------------------------- -BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s -BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s -BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s -``` +In C++11 mode, a ranged-based for loop should be used in preference to +the `KeepRunning` loop for running the benchmarks. For example: -The JSON format outputs human readable json split into two top level attributes. -The `context` attribute contains information about the run in general, including -information about the CPU and the date. -The `benchmarks` attribute contains a list of every benchmark run. Example json -output looks like: -```json -{ - "context": { - "date": "2015/03/17-18:40:25", - "num_cpus": 40, - "mhz_per_cpu": 2801, - "cpu_scaling_enabled": false, - "build_type": "debug" - }, - "benchmarks": [ - { - "name": "BM_SetInsert/1024/1", - "iterations": 94877, - "real_time": 29275, - "cpu_time": 29836, - "bytes_per_second": 134066, - "items_per_second": 33516 - }, - { - "name": "BM_SetInsert/1024/8", - "iterations": 21609, - "real_time": 32317, - "cpu_time": 32429, - "bytes_per_second": 986770, - "items_per_second": 246693 - }, - { - "name": "BM_SetInsert/1024/10", - "iterations": 21393, - "real_time": 32724, - "cpu_time": 33355, - "bytes_per_second": 1199226, - "items_per_second": 299807 - } - ] +```c++ +static void BM_Fast(benchmark::State &state) { + for (auto _ : state) { + FastOperation(); + } } +BENCHMARK(BM_Fast); ``` -The CSV format outputs comma-separated values. The `context` is output on stderr -and the CSV itself on stdout. Example CSV output looks like: -``` -name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label -"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, -"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, -"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, -``` - -### Output Files -The library supports writing the output of the benchmark to a file specified -by `--benchmark_out=`. The format of the output can be specified -using `--benchmark_out_format={json|console|csv}`. Specifying -`--benchmark_out` does not suppress the console output. - -## Result comparison - -It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) +The reason the ranged-for loop is faster than using `KeepRunning`, is +because `KeepRunning` requires a memory load and store of the iteration count +ever iteration, whereas the ranged-for variant is able to keep the iteration count +in a register. -## Debug vs Release -By default, benchmark builds as a debug library. You will see a warning in the -output when this is the case. To build it as a release library instead, use: +For example, an empty inner loop of using the ranged-based for method looks like: -``` -cmake -DCMAKE_BUILD_TYPE=Release +```asm +# Loop Init + mov rbx, qword ptr [r14 + 104] + call benchmark::State::StartKeepRunning() + test rbx, rbx + je .LoopEnd +.LoopHeader: # =>This Inner Loop Header: Depth=1 + add rbx, -1 + jne .LoopHeader +.LoopEnd: ``` -To enable link-time optimisation, use +Compared to an empty `KeepRunning` loop, which looks like: -``` -cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true +```asm +.LoopHeader: # in Loop: Header=BB0_3 Depth=1 + cmp byte ptr [rbx], 1 + jne .LoopInit +.LoopBody: # =>This Inner Loop Header: Depth=1 + mov rax, qword ptr [rbx + 8] + lea rcx, [rax + 1] + mov qword ptr [rbx + 8], rcx + cmp rax, qword ptr [rbx + 104] + jb .LoopHeader + jmp .LoopEnd +.LoopInit: + mov rdi, rbx + call benchmark::State::StartKeepRunning() + jmp .LoopBody +.LoopEnd: ``` -If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake -cache variables, if autodetection fails. - -If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, -`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. - -## Compiler Support - -Google Benchmark uses C++11 when building the library. As such we require -a modern C++ toolchain, both compiler and standard library. - -The following minimum versions are strongly recommended build the library: - -* GCC 4.8 -* Clang 3.4 -* Visual Studio 2013 -* Intel 2015 Update 1 - -Anything older *may* work. +Unless C++03 compatibility is required, the ranged-for variant of writing +the benchmark loop should be preferred. -Note: Using the library and its headers in C++03 is supported. C++11 is only -required to build the library. + -## Disable CPU frequency scaling +### Disabling CPU Frequency Scaling If you see this error: ``` ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. From 4b771940329acf371037e55b88730cdf9fbe391e Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 30 Apr 2019 13:36:29 +0300 Subject: [PATCH 144/330] CMake: codedrop of googletest cmake magic from me (#809) https://github.com/google/benchmark/pull/801 is stuck with some cryptic cmake failure due to some linking issue between googletest and threading libraries. I suspect that is mostly happening because of the, uhm, intentionally extremely twisted-in-the-brains approach that is being used to actually build the library as part of the buiild, except without actually building it as part of the build. If we do actually build it as part of the build, then all the transitive dependencies should magically be in order, and maybe everything will just work. This new version of cmake magic was written by me in https://github.com/darktable-org/rawspeed/blob/0e22f085c57cd629983c7351c598abd744b421b5/cmake/Modules/GoogleTest.cmake.in https://github.com/darktable-org/rawspeed/blob/0e22f085c57cd629983c7351c598abd744b421b5/cmake/Modules/GoogleTest.cmake, based on the official googletest docs and LOTS of experimentation. --- CMakeLists.txt | 2 +- cmake/GoogleTest.cmake | 40 ++++++++++++++ cmake/GoogleTest.cmake.in | 58 +++++++++++++++++++ cmake/HandleGTest.cmake | 113 -------------------------------------- test/CMakeLists.txt | 8 +-- 5 files changed, 100 insertions(+), 121 deletions(-) create mode 100644 cmake/GoogleTest.cmake create mode 100644 cmake/GoogleTest.cmake.in delete mode 100644 cmake/HandleGTest.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e51cb38533..5797f87fa1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -268,7 +268,7 @@ add_subdirectory(src) if (BENCHMARK_ENABLE_TESTING) enable_testing() if (BENCHMARK_ENABLE_GTEST_TESTS) - include(HandleGTest) + include(GoogleTest) endif() add_subdirectory(test) endif() diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake new file mode 100644 index 0000000000..6b98c91384 --- /dev/null +++ b/cmake/GoogleTest.cmake @@ -0,0 +1,40 @@ +# Download and unpack googletest at configure time +set(GOOGLETEST_PREFIX "${benchmark_BINARY_DIR}/third_party/googletest") +configure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY) + +execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" + -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH="${CMAKE_CURRENT_SOURCE_DIR}/googletest" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${GOOGLETEST_PREFIX} +) + +if(result) + message(FATAL_ERROR "CMake step for googletest failed: ${result}") +endif() + +execute_process( + COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${GOOGLETEST_PREFIX} +) + +if(result) + message(FATAL_ERROR "Build step for googletest failed: ${result}") +endif() + +# Prevent overriding the parent project's compiler/linker +# settings on Windows +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) + +# Add googletest directly to our build. This defines +# the gtest and gtest_main targets. +add_subdirectory(${GOOGLETEST_SOURCE_DIR} + ${GOOGLETEST_BINARY_DIR} + EXCLUDE_FROM_ALL) + +set_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $) +set_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $) +set_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $) +set_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $) diff --git a/cmake/GoogleTest.cmake.in b/cmake/GoogleTest.cmake.in new file mode 100644 index 0000000000..28818ee293 --- /dev/null +++ b/cmake/GoogleTest.cmake.in @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 2.8.12) + +project(googletest-download NONE) + +# Enable ExternalProject CMake module +include(ExternalProject) + +option(ALLOW_DOWNLOADING_GOOGLETEST "If googletest src tree is not found in location specified by GOOGLETEST_PATH, do fetch the archive from internet" OFF) +set(GOOGLETEST_PATH "/usr/src/googletest" CACHE PATH + "Path to the googletest root tree. Should contain googletest and googlemock subdirs. And CMakeLists.txt in root, and in both of these subdirs") + +# Download and install GoogleTest + +message(STATUS "Looking for Google Test sources") +message(STATUS "Looking for Google Test sources in ${GOOGLETEST_PATH}") +if(EXISTS "${GOOGLETEST_PATH}" AND IS_DIRECTORY "${GOOGLETEST_PATH}" AND EXISTS "${GOOGLETEST_PATH}/CMakeLists.txt" AND + EXISTS "${GOOGLETEST_PATH}/googletest" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googletest" AND EXISTS "${GOOGLETEST_PATH}/googletest/CMakeLists.txt" AND + EXISTS "${GOOGLETEST_PATH}/googlemock" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googlemock" AND EXISTS "${GOOGLETEST_PATH}/googlemock/CMakeLists.txt") + message(STATUS "Found Google Test in ${GOOGLETEST_PATH}") + + ExternalProject_Add( + googletest + PREFIX "${CMAKE_BINARY_DIR}" + DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" + SOURCE_DIR "${GOOGLETEST_PATH}" # use existing src dir. + BINARY_DIR "${CMAKE_BINARY_DIR}/build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) +else() + if(NOT ALLOW_DOWNLOADING_GOOGLETEST) + message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable ALLOW_DOWNLOADING_GOOGLETEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") + else() + message(WARNING "Did not find Google Test sources! Fetching from web...") + ExternalProject_Add( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + PREFIX "${CMAKE_BINARY_DIR}" + STAMP_DIR "${CMAKE_BINARY_DIR}/stamp" + DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download" + SOURCE_DIR "${CMAKE_BINARY_DIR}/src" + BINARY_DIR "${CMAKE_BINARY_DIR}/build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) + endif() +endif() + +ExternalProject_Get_Property(googletest SOURCE_DIR BINARY_DIR) +file(WRITE googletest-paths.cmake +"set(GOOGLETEST_SOURCE_DIR \"${SOURCE_DIR}\") +set(GOOGLETEST_BINARY_DIR \"${BINARY_DIR}\") +") diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake deleted file mode 100644 index b9c14436db..0000000000 --- a/cmake/HandleGTest.cmake +++ /dev/null @@ -1,113 +0,0 @@ - -include(split_list) - -macro(build_external_gtest) - include(ExternalProject) - set(GTEST_FLAGS "") - if (BENCHMARK_USE_LIBCXX) - if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - list(APPEND GTEST_FLAGS -stdlib=libc++) - else() - message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++") - endif() - endif() - if (BENCHMARK_BUILD_32_BITS) - list(APPEND GTEST_FLAGS -m32) - endif() - if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "") - list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS}) - endif() - string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE) - if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE") - set(GTEST_BUILD_TYPE "DEBUG") - endif() - # FIXME: Since 10/Feb/2017 the googletest trunk has had a bug where - # -Werror=unused-function fires during the build on OS X. This is a temporary - # workaround to keep our travis bots from failing. It should be removed - # once gtest is fixed. - if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") - list(APPEND GTEST_FLAGS "-Wno-unused-function") - endif() - split_list(GTEST_FLAGS) - set(EXCLUDE_FROM_ALL_OPT "") - set(EXCLUDE_FROM_ALL_VALUE "") - if (${CMAKE_VERSION} VERSION_GREATER "3.0.99") - set(EXCLUDE_FROM_ALL_OPT "EXCLUDE_FROM_ALL") - set(EXCLUDE_FROM_ALL_VALUE "ON") - endif() - ExternalProject_Add(googletest - ${EXCLUDE_FROM_ALL_OPT} ${EXCLUDE_FROM_ALL_VALUE} - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG master - PREFIX "${CMAKE_BINARY_DIR}/googletest" - INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest" - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE} - -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER} - -DCMAKE_INSTALL_PREFIX:PATH= - -DCMAKE_INSTALL_LIBDIR:PATH=/lib - -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS} - -Dgtest_force_shared_crt:BOOL=ON - ) - - ExternalProject_Get_Property(googletest install_dir) - set(GTEST_INCLUDE_DIRS ${install_dir}/include) - file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS}) - - set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}") - if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG") - set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}") - endif() - - # Use gmock_main instead of gtest_main because it initializes gtest as well. - # Note: The libraries are listed in reverse order of their dependancies. - foreach(LIB gtest gmock gmock_main) - add_library(${LIB} UNKNOWN IMPORTED) - set_target_properties(${LIB} PROPERTIES - IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}${LIB}${LIB_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIRS} - INTERFACE_LINK_LIBRARIES "${GTEST_BOTH_LIBRARIES}" - ) - add_dependencies(${LIB} googletest) - list(APPEND GTEST_BOTH_LIBRARIES ${LIB}) - endforeach() -endmacro(build_external_gtest) - -if (BENCHMARK_ENABLE_GTEST_TESTS) - if (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest) - set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest") - set(INSTALL_GTEST OFF CACHE INTERNAL "") - set(INSTALL_GMOCK OFF CACHE INTERNAL "") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/googletest) - set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main) - foreach(HEADER test mock) - # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we - # have to add the paths ourselves. - set(HFILE g${HEADER}/g${HEADER}.h) - set(HPATH ${GTEST_ROOT}/google${HEADER}/include) - find_path(HEADER_PATH_${HEADER} ${HFILE} - NO_DEFAULT_PATHS - HINTS ${HPATH} - ) - if (NOT HEADER_PATH_${HEADER}) - message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}") - endif() - list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}}) - endforeach() - elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES) - build_external_gtest() - else() - find_package(GTest REQUIRED) - find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h - HINTS ${GTEST_INCLUDE_DIRS}) - if (NOT GMOCK_INCLUDE_DIRS) - message(FATAL_ERROR "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}") - endif() - set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS}) - # FIXME: We don't currently require the gmock library to build the tests, - # and it's likely we won't find it, so we don't try. As long as we've - # found the gmock/gmock.h header and gtest_main that should be good enough. - endif() -endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d3a07639bf..030f35aae3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -178,14 +178,8 @@ add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(compile_gtest name) add_executable(${name} "${name}.cc") - if (TARGET googletest) - add_dependencies(${name} googletest) - endif() - if (GTEST_INCLUDE_DIRS) - target_include_directories(${name} PRIVATE ${GTEST_INCLUDE_DIRS}) - endif() target_link_libraries(${name} benchmark - ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) + gmock_main ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_gtest) macro(add_gtest name) From 727a81aabd1354c07eac652da7671b0b7192bfa7 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 30 Apr 2019 19:32:38 +0300 Subject: [PATCH 145/330] CMake: avoid incorrect extra "" around the GOOGLETEST_PATH. It was looking in `"<...>/googlebenchmark/build/third_party/googletest"`, with these extra `"` quotes, and wasn't finding anything.. --- cmake/GoogleTest.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake index 6b98c91384..fb7c6be25e 100644 --- a/cmake/GoogleTest.cmake +++ b/cmake/GoogleTest.cmake @@ -2,8 +2,9 @@ set(GOOGLETEST_PREFIX "${benchmark_BINARY_DIR}/third_party/googletest") configure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY) +set(GOOGLETEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/googletest") # Mind the quotes execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" - -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH="${CMAKE_CURRENT_SOURCE_DIR}/googletest" . + -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH=${GOOGLETEST_PATH} . RESULT_VARIABLE result WORKING_DIRECTORY ${GOOGLETEST_PREFIX} ) From 7da47d039dde28ad96e1e62ddfec85d746134738 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 16 Apr 2019 12:12:37 +0100 Subject: [PATCH 146/330] Travis-ci: more correctly pass -m32 to 32-bit jobs Generally we can't just add -m32 in the middle of the build. It currently just happens to work, but that is not guaranteed. --- .travis.yml | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51d652cf98..55ea6734e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,13 +23,25 @@ matrix: apt: packages: - g++-multilib - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON + - libc6:i386 + env: + - COMPILER=g++ + - C_COMPILER=gcc + - BUILD_TYPE=Debug + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - compiler: gcc addons: apt: packages: - g++-multilib - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON + - libc6:i386 + env: + - COMPILER=g++ + - C_COMPILER=gcc + - BUILD_TYPE=Release + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - compiler: gcc env: - INSTALL_GCC6_FROM_PPA=1 @@ -51,7 +63,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - - EXTRA_FLAGS="-stdlib=libc++" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - compiler: clang dist: xenial addons: @@ -62,7 +74,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - - EXTRA_FLAGS="-stdlib=libc++" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -71,12 +83,14 @@ matrix: packages: - clang-3.8 - g++-multilib + - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-stdlib=libc++ -m32" + - EXTRA_FLAGS="-m32" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -85,12 +99,14 @@ matrix: packages: - clang-3.8 - g++-multilib + - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-stdlib=libc++ -m32" + - EXTRA_FLAGS="-m32" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ libc++, ASAN, UBSAN - compiler: clang dist: xenial @@ -103,7 +119,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN - compiler: clang @@ -117,7 +134,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ libc++ and MSAN - compiler: clang dist: xenial @@ -130,7 +148,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - os: osx osx_image: xcode8.3 compiler: clang @@ -145,7 +164,10 @@ matrix: osx_image: xcode8.3 compiler: clang env: - - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON + - COMPILER=clang++ + - BUILD_TYPE=Release + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - os: osx osx_image: xcode8.3 compiler: gcc @@ -202,7 +224,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... From 94115f441909ad7478d1439c01395a9b78f8a23e Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 30 Apr 2019 21:07:01 +0300 Subject: [PATCH 147/330] Revert "Travis-ci: more correctly pass -m32 to 32-bit jobs" Very interesting, i was completely not expecting to see that cmake failure from https://github.com/google/benchmark/pull/801 *here*. This reverts commit 7da47d039dde28ad96e1e62ddfec85d746134738. --- .travis.yml | 44 +++++++++++--------------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55ea6734e0..51d652cf98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,25 +23,13 @@ matrix: apt: packages: - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Debug - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON - compiler: gcc addons: apt: packages: - g++-multilib - - libc6:i386 - env: - - COMPILER=g++ - - C_COMPILER=gcc - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON - compiler: gcc env: - INSTALL_GCC6_FROM_PPA=1 @@ -63,7 +51,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++" - compiler: clang dist: xenial addons: @@ -74,7 +62,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -83,14 +71,12 @@ matrix: packages: - clang-3.8 - g++-multilib - - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -99,14 +85,12 @@ matrix: packages: - clang-3.8 - g++-multilib - - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -m32" # Clang w/ libc++, ASAN, UBSAN - compiler: clang dist: xenial @@ -119,8 +103,7 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN - compiler: clang @@ -134,8 +117,7 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" # Clang w/ libc++ and MSAN - compiler: clang dist: xenial @@ -148,8 +130,7 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - - EXTRA_CXX_FLAGS="-stdlib=libc++" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - os: osx osx_image: xcode8.3 compiler: clang @@ -164,10 +145,7 @@ matrix: osx_image: xcode8.3 compiler: clang env: - - COMPILER=clang++ - - BUILD_TYPE=Release - - BUILD_32_BITS=ON - - EXTRA_FLAGS="-m32" + - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON - os: osx osx_image: xcode8.3 compiler: gcc @@ -224,7 +202,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... From 13b8bdc2b51805c3ccd666a511ae092784b1c20e Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 1 May 2019 09:06:12 +0100 Subject: [PATCH 148/330] Bump required cmake version from 2.x to 3.x (#801) --- .travis.yml | 46 ++++++++++++++++++++++++++++++++++------------ CMakeLists.txt | 2 +- README.md | 3 +++ dependencies.md | 18 ++++++++++++++++++ src/CMakeLists.txt | 6 ++++++ 5 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 dependencies.md diff --git a/.travis.yml b/.travis.yml index 51d652cf98..6b6cfc7046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,13 +23,25 @@ matrix: apt: packages: - g++-multilib - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON + - libc6:i386 + env: + - COMPILER=g++ + - C_COMPILER=gcc + - BUILD_TYPE=Debug + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - compiler: gcc addons: apt: packages: - g++-multilib - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON + - libc6:i386 + env: + - COMPILER=g++ + - C_COMPILER=gcc + - BUILD_TYPE=Release + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - compiler: gcc env: - INSTALL_GCC6_FROM_PPA=1 @@ -51,7 +63,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - - EXTRA_FLAGS="-stdlib=libc++" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - compiler: clang dist: xenial addons: @@ -62,7 +74,7 @@ matrix: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - - EXTRA_FLAGS="-stdlib=libc++" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -71,12 +83,14 @@ matrix: packages: - clang-3.8 - g++-multilib + - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-stdlib=libc++ -m32" + - EXTRA_FLAGS="-m32" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ 32bit libc++ - compiler: clang dist: xenial @@ -85,12 +99,14 @@ matrix: packages: - clang-3.8 - g++-multilib + - libc6:i386 env: - INSTALL_GCC6_FROM_PPA=1 - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release - LIBCXX_BUILD=1 - BUILD_32_BITS=ON - - EXTRA_FLAGS="-stdlib=libc++ -m32" + - EXTRA_FLAGS="-m32" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ libc++, ASAN, UBSAN - compiler: clang dist: xenial @@ -103,7 +119,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - UBSAN_OPTIONS=print_stacktrace=1 # Clang w/ libc++ and MSAN - compiler: clang @@ -117,7 +134,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" + - EXTRA_CXX_FLAGS="-stdlib=libc++" # Clang w/ libc++ and MSAN - compiler: clang dist: xenial @@ -130,7 +148,8 @@ matrix: - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread - ENABLE_SANITIZER=1 - - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" + - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" + - EXTRA_CXX_FLAGS="-stdlib=libc++" - os: osx osx_image: xcode8.3 compiler: clang @@ -145,7 +164,10 @@ matrix: osx_image: xcode8.3 compiler: clang env: - - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON + - COMPILER=clang++ + - BUILD_TYPE=Release + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-m32" - os: osx osx_image: xcode8.3 compiler: gcc @@ -192,7 +214,7 @@ install: fi - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo apt-get update -qq; - sudo apt-get install -qq unzip; + sudo apt-get install -qq unzip cmake3; wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh; travis_wait sudo bash bazel-installer.sh; fi @@ -202,7 +224,7 @@ install: fi script: - - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} .. - make - ctest -C ${BUILD_TYPE} --output-on-failure - bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/... diff --git a/CMakeLists.txt b/CMakeLists.txt index 5797f87fa1..856bc98bb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 2.8.12) +cmake_minimum_required (VERSION 3.5.1) foreach(p CMP0048 # OK to clear PROJECT_VERSION on project() diff --git a/README.md b/README.md index 251c5cd673..45e4158843 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ The following minimum versions are required to build the library: This describes the installation process using cmake. As pre-requisites, you'll need git and cmake installed. +_See [dependencies.md](dependencies.md) for more details regarding supported +versions of build tools._ + ```bash # Check out the library. $ git clone https://github.com/google/benchmark.git diff --git a/dependencies.md b/dependencies.md new file mode 100644 index 0000000000..6289b4e354 --- /dev/null +++ b/dependencies.md @@ -0,0 +1,18 @@ +# Build tool dependency policy + +To ensure the broadest compatibility when building the benchmark library, but +still allow forward progress, we require any build tooling to be available for: + +* Debian stable AND +* The last two Ubuntu LTS releases AND + +Currently, this means using build tool versions that are available for Ubuntu +16.04 (Xenial), Ubuntu 18.04 (Bionic), and Debian stretch. + +_Note, [travis](.travis.yml) runs under Ubuntu 14.04 (Trusty) for linux builds._ + +## cmake +The current supported version is cmake 3.5.1 as of 2018-06-06. + +_Note, this version is also available for Ubuntu 14.04, the previous Ubuntu LTS +release, as `cmake3`._ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 977474f43f..4c75a9dedd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,12 @@ if(LIBRT) target_link_libraries(benchmark ${LIBRT}) endif() +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) +if(NOT CMAKE_THREAD_LIBS_INIT AND "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" MATCHES ".*-fsanitize=[^ ]*address.*") + message(WARNING "CMake's FindThreads.cmake did not fail, but CMAKE_THREAD_LIBS_INIT ended up being empty. This was fixed in https://github.com/Kitware/CMake/commit/d53317130e84898c5328c237186dbd995aaf1c12 Let's guess that -pthread is sufficient.") + target_link_libraries(benchmark -pthread) +endif() + # We need extra libraries on Windows if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_link_libraries(benchmark Shlwapi) From 823d24630db0f340914bb011f1ba31657348e510 Mon Sep 17 00:00:00 2001 From: Lockywolf Date: Wed, 1 May 2019 16:13:33 +0800 Subject: [PATCH 149/330] Add support for GNU Install Dirs from GNU Coding Standards. Fixes #807 (#808) * Add support for GNU Install Dirs from GNU Coding Standards * src/CMakeLists.txt: Added support for setting the standard variables, such as CMAKE_INSTALL_BINDIR. * Replace install destinations by the ones from GNU Coding Standards. * Set the default .cmake and .pc default path. --- src/CMakeLists.txt | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c75a9dedd..1a8c602d59 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ # Allow the source files to find headers in src/ +include(GNUInstallDirs) include_directories(${PROJECT_SOURCE_DIR}/src) if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) @@ -61,11 +62,6 @@ target_include_directories(benchmark PUBLIC ) target_link_libraries(benchmark_main benchmark) -set(include_install_dir "include") -set(lib_install_dir "lib/") -set(bin_install_dir "bin/") -set(config_install_dir "lib/cmake/${PROJECT_NAME}") -set(pkgconfig_install_dir "lib/pkgconfig") set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") @@ -89,26 +85,26 @@ if (BENCHMARK_ENABLE_INSTALL) install( TARGETS benchmark benchmark_main EXPORT ${targets_export_name} - ARCHIVE DESTINATION ${lib_install_dir} - LIBRARY DESTINATION ${lib_install_dir} - RUNTIME DESTINATION ${bin_install_dir} - INCLUDES DESTINATION ${include_install_dir}) + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install( DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" - DESTINATION ${include_install_dir} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h") install( FILES "${project_config}" "${version_config}" - DESTINATION "${config_install_dir}") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") install( FILES "${pkg_config}" - DESTINATION "${pkgconfig_install_dir}") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") install( EXPORT "${targets_export_name}" NAMESPACE "${namespace}" - DESTINATION "${config_install_dir}") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") endif() From 7d856b03047e7bf09ccc9e9878e299493541b468 Mon Sep 17 00:00:00 2001 From: Shuo Chen Date: Wed, 1 May 2019 13:00:10 -0700 Subject: [PATCH 150/330] If gtest targets are already defined, use them. (#777) This allows a project to include both googletest and benchmark as top-level git submodule. This allows incorporating Benchmark to an existing CMake project that already incorporates googletest. https://github.com/google/googletest/blob/master/googletest/README.md#incorporating-into-an-existing-cmake-project https://github.com/abseil/abseil-cpp/tree/master/CMake#incorporating-abseil-into-a-cmake-project --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 856bc98bb2..74b99341b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,7 +267,9 @@ add_subdirectory(src) if (BENCHMARK_ENABLE_TESTING) enable_testing() - if (BENCHMARK_ENABLE_GTEST_TESTS) + if (BENCHMARK_ENABLE_GTEST_TESTS AND + NOT (TARGET gtest AND TARGET gtest_main AND + TARGET gmock AND TARGET gmock_main)) include(GoogleTest) endif() add_subdirectory(test) From 33d440465041fe83b831f537b9646756655806f0 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 8 May 2019 02:06:50 +0300 Subject: [PATCH 151/330] Don't read CMAKE_BUILD_TYPE if it is not there (#811) Weird, but seems consistent with the rest of cmake here. --- src/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a8c602d59..b47de6791c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,7 +34,9 @@ if(LIBRT) target_link_libraries(benchmark ${LIBRT}) endif() -string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) +if(CMAKE_BUILD_TYPE) + string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER) +endif() if(NOT CMAKE_THREAD_LIBS_INIT AND "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" MATCHES ".*-fsanitize=[^ ]*address.*") message(WARNING "CMake's FindThreads.cmake did not fail, but CMAKE_THREAD_LIBS_INIT ended up being empty. This was fixed in https://github.com/Kitware/CMake/commit/d53317130e84898c5328c237186dbd995aaf1c12 Let's guess that -pthread is sufficient.") target_link_libraries(benchmark -pthread) From b988639f316ed0aca69297effbf67015c3bf0640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Janiszewski?= Date: Fri, 10 May 2019 00:22:13 +0200 Subject: [PATCH 152/330] Fix compilation for Android (#816) Android doesn't support `getloadavg` --- src/sysinfo.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 01ecfad028..28126470ba 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -658,9 +658,9 @@ double GetCPUCyclesPerSecond() { } std::vector GetLoadAvg() { -#if defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ +#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ - defined BENCHMARK_OS_OPENBSD + defined BENCHMARK_OS_OPENBSD) && !defined(__ANDROID__) constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); From 12c978c51343a0376903a99c6435bdf52138a952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Janiszewski?= Date: Fri, 10 May 2019 18:56:08 +0200 Subject: [PATCH 153/330] Mark CMake project as C++-only (#681) This will make CMake skip all the checks for C compiler. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 74b99341b3..41b3b66851 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ foreach(p endif() endforeach() -project (benchmark) +project (benchmark CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) From 2e7203aa94bf49cdfb60e58a7694784260bc6cc5 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 11 May 2019 18:56:57 +0300 Subject: [PATCH 154/330] CMake: check CMAKE_CXX_COMPILER_ID, not CMAKE_C_COMPILER_ID This may or may not have gotten broken now that benchmark was marked as CXX-only. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41b3b66851..9db1361212 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,7 +191,7 @@ else() # Link time optimisation if (BENCHMARK_ENABLE_LTO) add_cxx_compiler_flag(-flto) - if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") find_program(GCC_AR gcc-ar) if (GCC_AR) set(CMAKE_AR ${GCC_AR}) @@ -200,7 +200,7 @@ else() if (GCC_RANLIB) set(CMAKE_RANLIB ${GCC_RANLIB}) endif() - elseif("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") + elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") include(llvm-toolchain) endif() endif() From f92903cc5338daed898242f22015d8426c065770 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 13 May 2019 12:33:11 +0300 Subject: [PATCH 155/330] Iteration counts should be `uint64_t` globally. (#817) This is a shameless rip-off of https://github.com/google/benchmark/pull/646 I did promise to look into why that proposed PR was producing so much worse assembly, and so i finally did. The reason is - that diff changes `size_t` (unsigned) to `int64_t` (signed). There is this nice little `assert`: https://github.com/google/benchmark/blob/7a1c37028359ca9d386d719a6ad527743cf1b753/include/benchmark/benchmark.h#L744 It ensures that we didn't magically decide to advance our iterator when we should have finished benchmarking. When `cached_` was unsigned, the `assert` was `cached_ UGT 0`. But we only ever get to that `assert` if `cached_ NE 0`, and naturally if `cached_` is not `0`, then it is bigger than `0`, so the `assert` is tautological, and gets folded away. But now that `cached_` became signed, the assert became `cached_ SGT 0`. And we still only know that `cached_ NE 0`, so the assert can't be optimized out, or at least it doesn't currently. Regardless of whether or not that is a bug in itself, that particular diff would have regressed the normal 64-bit systems, by halving the maximal iteration space (since we go from unsigned counter to signed one, of the same bit-width), which seems like a bug. And just so it happens, fixing *this* bug, fixes the other bug. This produces fully (bit-by-bit) identical state_assembly_test.s The filecheck change is actually needed regardless of this patch, else this test does not pass for me even without this diff. --- include/benchmark/benchmark.h | 38 +++++++++++++++++------------------ src/benchmark.cc | 4 ++-- src/benchmark_api_internal.cc | 6 +++--- src/benchmark_api_internal.h | 4 ++-- src/benchmark_register.cc | 2 +- src/benchmark_runner.cc | 23 +++++++++++---------- src/complexity.cc | 15 ++++++++------ src/counter.cc | 5 +++-- src/counter.h | 3 ++- src/json_reporter.cc | 6 ++++++ src/thread_manager.h | 2 +- test/basic_test.cc | 8 ++++---- test/complexity_test.cc | 12 ++++++----- test/cxx03_test.cc | 2 +- test/state_assembly_test.cc | 2 +- 15 files changed, 73 insertions(+), 59 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 8329368571..6cb96f546d 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -56,8 +56,7 @@ static void BM_memcpy(benchmark::State& state) { memset(src, 'x', state.range(0)); for (auto _ : state) memcpy(dst, src, state.range(0)); - state.SetBytesProcessed(int64_t(state.iterations()) * - int64_t(state.range(0))); + state.SetBytesProcessed(state.iterations() * state.range(0)); delete[] src; delete[] dst; } BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); @@ -122,8 +121,7 @@ template int BM_Sequential(benchmark::State& state) { q.Wait(&v); } // actually messages, not bytes: - state.SetBytesProcessed( - static_cast(state.iterations())*state.range(0)); + state.SetBytesProcessed(state.iterations() * state.range(0)); } BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); @@ -413,9 +411,11 @@ enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; // calculated automatically to the best fit. enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; +typedef uint64_t IterationCount; + // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(int64_t); +typedef double(BigOFunc)(IterationCount); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -488,7 +488,7 @@ class State { // while (state.KeepRunningBatch(1000)) { // // process 1000 elements // } - bool KeepRunningBatch(size_t n); + bool KeepRunningBatch(IterationCount n); // REQUIRES: timer is running and 'SkipWithError(...)' has not been called // by the current thread. @@ -627,7 +627,7 @@ class State { int64_t range_y() const { return range(1); } BENCHMARK_ALWAYS_INLINE - size_t iterations() const { + IterationCount iterations() const { if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { return 0; } @@ -638,15 +638,15 @@ class State { : // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. // May be larger than max_iterations. - size_t total_iterations_; + IterationCount total_iterations_; // When using KeepRunningBatch(), batch_leftover_ holds the number of // iterations beyond max_iters that were run. Used to track // completed_iterations_ accurately. - size_t batch_leftover_; + IterationCount batch_leftover_; public: - const size_t max_iterations; + const IterationCount max_iterations; private: bool started_; @@ -667,14 +667,14 @@ class State { const int threads; private: - State(size_t max_iters, const std::vector& ranges, int thread_i, - int n_threads, internal::ThreadTimer* timer, + State(IterationCount max_iters, const std::vector& ranges, + int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. - bool KeepRunningInternal(size_t n, bool is_batch); + bool KeepRunningInternal(IterationCount n, bool is_batch); void FinishKeepRunning(); internal::ThreadTimer* timer_; internal::ThreadManager* manager_; @@ -686,11 +686,11 @@ inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { return KeepRunningInternal(1, /*is_batch=*/false); } -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(size_t n) { +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) { return KeepRunningInternal(n, /*is_batch=*/true); } -inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(size_t n, +inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, bool is_batch) { // total_iterations_ is set to 0 by the constructor, and always set to a // nonzero value by StartKepRunning(). @@ -754,7 +754,7 @@ struct State::StateIterator { } private: - size_t cached_; + IterationCount cached_; State* const parent_; }; @@ -858,7 +858,7 @@ class Benchmark { // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. - Benchmark* Iterations(size_t n); + Benchmark* Iterations(IterationCount n); // Specify the amount of times to repeat this benchmark. This option overrides // the `benchmark_repetitions` flag. @@ -957,7 +957,7 @@ class Benchmark { TimeUnit time_unit_; int range_multiplier_; double min_time_; - size_t iterations_; + IterationCount iterations_; int repetitions_; bool measure_process_cpu_time_; bool use_real_time_; @@ -1375,7 +1375,7 @@ class BenchmarkReporter { bool error_occurred; std::string error_message; - int64_t iterations; + IterationCount iterations; int64_t threads; int64_t repetition_index; int64_t repetitions; diff --git a/src/benchmark.cc b/src/benchmark.cc index c76346c6c2..29bfa3512f 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -121,8 +121,8 @@ void UseCharPointer(char const volatile*) {} } // namespace internal -State::State(size_t max_iters, const std::vector& ranges, int thread_i, - int n_threads, internal::ThreadTimer* timer, +State::State(IterationCount max_iters, const std::vector& ranges, + int thread_i, int n_threads, internal::ThreadTimer* timer, internal::ThreadManager* manager) : total_iterations_(0), batch_leftover_(0), diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 8d3108363b..d468a257e3 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -3,9 +3,9 @@ namespace benchmark { namespace internal { -State BenchmarkInstance::Run( - size_t iters, int thread_id, internal::ThreadTimer* timer, - internal::ThreadManager* manager) const { +State BenchmarkInstance::Run(IterationCount iters, int thread_id, + internal::ThreadTimer* timer, + internal::ThreadManager* manager) const { State st(iters, arg, thread_id, threads, timer, manager); benchmark->Run(st); return st; diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 19c6d4f7d2..264eff95c5 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -32,10 +32,10 @@ struct BenchmarkInstance { bool last_benchmark_instance; int repetitions; double min_time; - size_t iterations; + IterationCount iterations; int threads; // Number of concurrent threads to us - State Run(size_t iters, int thread_id, internal::ThreadTimer* timer, + State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager) const; }; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 3ffd734550..6696c382b8 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -376,7 +376,7 @@ Benchmark* Benchmark::MinTime(double t) { return this; } -Benchmark* Benchmark::Iterations(size_t n) { +Benchmark* Benchmark::Iterations(IterationCount n) { CHECK(n > 0); CHECK(IsZero(min_time_)); iterations_ = n; diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index b1c0b8896a..0bae6a545e 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -59,11 +59,12 @@ MemoryManager* memory_manager = nullptr; namespace { -static const size_t kMaxIterations = 1000000000; +static constexpr IterationCount kMaxIterations = 1000000000; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, - const internal::ThreadManager::Result& results, size_t memory_iterations, + const internal::ThreadManager::Result& results, + IterationCount memory_iterations, const MemoryManager::Result& memory_result, double seconds, int64_t repetition_index) { // Create report about this benchmark run. @@ -109,8 +110,8 @@ BenchmarkReporter::Run CreateRunReport( // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into *total. -void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id, - ThreadManager* manager) { +void RunInThread(const BenchmarkInstance* b, IterationCount iters, + int thread_id, ThreadManager* manager) { internal::ThreadTimer timer( b->measure_process_cpu_time ? internal::ThreadTimer::CreateProcessCpuTime() @@ -187,13 +188,13 @@ class BenchmarkRunner { std::vector pool; - size_t iters; // preserved between repetitions! + IterationCount iters; // preserved between repetitions! // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. struct IterationResults { internal::ThreadManager::Result results; - size_t iters; + IterationCount iters; double seconds; }; IterationResults DoNIterations() { @@ -248,7 +249,7 @@ class BenchmarkRunner { return i; } - size_t PredictNumItersNeeded(const IterationResults& i) const { + IterationCount PredictNumItersNeeded(const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); @@ -262,10 +263,10 @@ class BenchmarkRunner { if (multiplier <= 1.0) multiplier = 2.0; // So what seems to be the sufficiently-large iteration count? Round up. - const size_t max_next_iters = + const IterationCount max_next_iters = 0.5 + std::max(multiplier * i.iters, i.iters + 1.0); // But we do have *some* sanity limits though.. - const size_t next_iters = std::min(max_next_iters, kMaxIterations); + const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; return next_iters; // round up before conversion to integer. @@ -319,11 +320,11 @@ class BenchmarkRunner { // Oh, one last thing, we need to also produce the 'memory measurements'.. MemoryManager::Result memory_result; - size_t memory_iterations = 0; + IterationCount memory_iterations = 0; if (memory_manager != nullptr) { // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. - memory_iterations = std::min(16, iters); + memory_iterations = std::min(16, iters); memory_manager->Start(); std::unique_ptr manager; manager.reset(new internal::ThreadManager(1)); diff --git a/src/complexity.cc b/src/complexity.cc index e65bd2ebdf..aeed67f0c7 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -29,20 +29,23 @@ BigOFunc* FittingCurve(BigO complexity) { static const double kLog2E = 1.44269504088896340736; switch (complexity) { case oN: - return [](int64_t n) -> double { return static_cast(n); }; + return [](IterationCount n) -> double { return static_cast(n); }; case oNSquared: - return [](int64_t n) -> double { return std::pow(n, 2); }; + return [](IterationCount n) -> double { return std::pow(n, 2); }; case oNCubed: - return [](int64_t n) -> double { return std::pow(n, 3); }; + return [](IterationCount n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](int64_t n) { return kLog2E * log(static_cast(n)); }; + return + [](IterationCount n) { return kLog2E * log(static_cast(n)); }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](int64_t n) { return kLog2E * n * log(static_cast(n)); }; + return [](IterationCount n) { + return kLog2E * n * log(static_cast(n)); + }; case o1: default: - return [](int64_t) { return 1.0; }; + return [](IterationCount) { return 1.0; }; } } diff --git a/src/counter.cc b/src/counter.cc index cb604e060b..c248ea110b 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -17,7 +17,7 @@ namespace benchmark { namespace internal { -double Finish(Counter const& c, int64_t iterations, double cpu_time, +double Finish(Counter const& c, IterationCount iterations, double cpu_time, double num_threads) { double v = c.value; if (c.flags & Counter::kIsRate) { @@ -35,7 +35,8 @@ double Finish(Counter const& c, int64_t iterations, double cpu_time, return v; } -void Finish(UserCounters* l, int64_t iterations, double cpu_time, double num_threads) { +void Finish(UserCounters* l, IterationCount iterations, double cpu_time, + double num_threads) { for (auto& c : *l) { c.second.value = Finish(c.second, iterations, cpu_time, num_threads); } diff --git a/src/counter.h b/src/counter.h index d884e50aa1..1ad46d4940 100644 --- a/src/counter.h +++ b/src/counter.h @@ -18,7 +18,8 @@ namespace benchmark { // these counter-related functions are hidden to reduce API surface. namespace internal { -void Finish(UserCounters* l, int64_t iterations, double time, double num_threads); +void Finish(UserCounters* l, IterationCount iterations, double time, + double num_threads); void Increment(UserCounters* l, UserCounters const& r); bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace internal diff --git a/src/json_reporter.cc b/src/json_reporter.cc index cf1de2547a..11db2b99d5 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -68,6 +68,12 @@ std::string FormatKV(std::string const& key, int64_t value) { return ss.str(); } +std::string FormatKV(std::string const& key, IterationCount value) { + std::stringstream ss; + ss << '"' << StrEscape(key) << "\": " << value; + return ss.str(); +} + std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << StrEscape(key) << "\": "; diff --git a/src/thread_manager.h b/src/thread_manager.h index 6e274c7ea6..1720281f0a 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -38,7 +38,7 @@ class ThreadManager { public: struct Result { - int64_t iterations = 0; + IterationCount iterations = 0; double real_time_used = 0; double cpu_time_used = 0; double manual_time_used = 0; diff --git a/test/basic_test.cc b/test/basic_test.cc index d07fbc00b1..5f3dd1a3ee 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -98,7 +98,7 @@ BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); void BM_KeepRunning(benchmark::State& state) { - size_t iter_count = 0; + benchmark::IterationCount iter_count = 0; assert(iter_count == state.iterations()); while (state.KeepRunning()) { ++iter_count; @@ -109,8 +109,8 @@ BENCHMARK(BM_KeepRunning); void BM_KeepRunningBatch(benchmark::State& state) { // Choose a prime batch size to avoid evenly dividing max_iterations. - const size_t batch_size = 101; - size_t iter_count = 0; + const benchmark::IterationCount batch_size = 101; + benchmark::IterationCount iter_count = 0; while (state.KeepRunningBatch(batch_size)) { iter_count += batch_size; } @@ -119,7 +119,7 @@ void BM_KeepRunningBatch(benchmark::State& state) { BENCHMARK(BM_KeepRunningBatch); void BM_RangedFor(benchmark::State& state) { - size_t iter_count = 0; + benchmark::IterationCount iter_count = 0; for (auto _ : state) { ++iter_count; } diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 4a62869248..d4febbbc15 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -66,9 +66,9 @@ void BM_Complexity_O1(benchmark::State& state) { } BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) { - return 1.0; -}); +BENCHMARK(BM_Complexity_O1) + ->Range(1, 1 << 18) + ->Complexity([](benchmark::IterationCount) { return 1.0; }); const char *one_test_name = "BM_Complexity_O1"; const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; @@ -121,7 +121,9 @@ BENCHMARK(BM_Complexity_O_N) BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) -> double { return static_cast(n); }); + ->Complexity([](benchmark::IterationCount n) -> double { + return static_cast(n); + }); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) @@ -160,7 +162,7 @@ BENCHMARK(BM_Complexity_O_N_log_N) BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) ->Range(1 << 10, 1 << 16) - ->Complexity([](int64_t n) { + ->Complexity([](benchmark::IterationCount n) { return kLog2E * n * log(static_cast(n)); }); BENCHMARK(BM_Complexity_O_N_log_N) diff --git a/test/cxx03_test.cc b/test/cxx03_test.cc index baa9ed9262..c4c9a52273 100644 --- a/test/cxx03_test.cc +++ b/test/cxx03_test.cc @@ -14,7 +14,7 @@ void BM_empty(benchmark::State& state) { while (state.KeepRunning()) { - volatile std::size_t x = state.iterations(); + volatile benchmark::IterationCount x = state.iterations(); ((void)x); } } diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc index abe9a4ddb5..7ddbb3b2a9 100644 --- a/test/state_assembly_test.cc +++ b/test/state_assembly_test.cc @@ -25,7 +25,7 @@ extern "C" int test_for_auto_loop() { for (auto _ : S) { // CHECK: .L[[LOOP_HEAD:[a-zA-Z0-9_]+]]: // CHECK-GNU-NEXT: subq $1, %rbx - // CHECK-CLANG-NEXT: {{(addq \$1,|incq)}} %rax + // CHECK-CLANG-NEXT: {{(addq \$1, %rax|incq %rax|addq \$-1, %rbx)}} // CHECK-NEXT: jne .L[[LOOP_HEAD]] benchmark::DoNotOptimize(x); } From 090faecb454fbd6e6e17a75ef8146acb037118d4 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 13 May 2019 22:41:42 +0300 Subject: [PATCH 156/330] Use IterationCount in one more place Found in -UNDEBUG build --- src/statistics.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/statistics.cc b/src/statistics.cc index 4dc2c95f31..bd5a3d6597 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -97,7 +97,7 @@ std::vector ComputeStats( // All repetitions should be run with the same number of iterations so we // can take this information from the first benchmark. - int64_t const run_iterations = reports.front().iterations; + const IterationCount run_iterations = reports.front().iterations; // create stats for user counters struct CounterStat { Counter c; From 04a9343fc9b7886591c65933b94d0cee7a915452 Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Wed, 26 Jun 2019 11:06:24 +0300 Subject: [PATCH 157/330] Make some functions const (#832) and ThreadManager ctor explicit. Reported by CppCheck. --- include/benchmark/benchmark.h | 2 +- src/thread_manager.h | 2 +- src/thread_timer.h | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 6cb96f546d..856605f25b 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -574,7 +574,7 @@ class State { void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() { return complexity_n_; } + int64_t complexity_length_n() const { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently diff --git a/src/thread_manager.h b/src/thread_manager.h index 1720281f0a..28e2dd53af 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -11,7 +11,7 @@ namespace internal { class ThreadManager { public: - ThreadManager(int num_threads) + explicit ThreadManager(int num_threads) : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { diff --git a/src/thread_timer.h b/src/thread_timer.h index fbd298d3bd..1703ca0d6f 100644 --- a/src/thread_timer.h +++ b/src/thread_timer.h @@ -43,19 +43,19 @@ class ThreadTimer { bool running() const { return running_; } // REQUIRES: timer is not running - double real_time_used() { + double real_time_used() const { CHECK(!running_); return real_time_used_; } // REQUIRES: timer is not running - double cpu_time_used() { + double cpu_time_used() const { CHECK(!running_); return cpu_time_used_; } // REQUIRES: timer is not running - double manual_time_used() { + double manual_time_used() const { CHECK(!running_); return manual_time_used_; } From 4abdfbb802d1b514703223f5f852ce4a507d32d2 Mon Sep 17 00:00:00 2001 From: Sam Elliott Date: Fri, 5 Jul 2019 09:28:17 +0100 Subject: [PATCH 158/330] Add RISC-V support in cycleclock::Now (#833) The RISC-V implementation of `cycleclock::Now` uses the user-space `rdcycle` instruction to query how many cycles have happened since the core started. The only complexity here is on 32-bit RISC-V, where `rdcycle` can only read the lower 32 bits of the 64-bit hardware counter. In this case, `rdcycleh` reads the higher 32 bits of the counter. We match the powerpc implementation to detect and correct for overflow in the high bits. --- src/cycleclock.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/cycleclock.h b/src/cycleclock.h index f5e37b011b..d5d62c4c7f 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -164,6 +164,21 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { uint64_t tsc; asm("stck %0" : "=Q"(tsc) : : "cc"); return tsc; +#elif defined(__riscv) // RISC-V + // Use RDCYCLE (and RDCYCLEH on riscv32) +#if __riscv_xlen == 32 + uint64_t cycles_low, cycles_hi0, cycles_hi1; + asm("rdcycleh %0" : "=r"(cycles_hi0)); + asm("rdcycle %0" : "=r"(cycles_lo)); + asm("rdcycleh %0" : "=r"(cycles_hi1)); + // This matches the PowerPC overflow detection, above + cycles_lo &= -static_cast(cycles_hi0 == cycles_hi1); + return (cycles_hi1 << 32) | cycles_lo; +#else + uint64_t cycles; + asm("rdcycle %0" : "=r"(cycles)); + return cycles; +#endif #else // The soft failover to a generic implementation is automatic only for ARM. // For other platforms the developer is expected to make an attempt to create From df4f9fe36232934eea06d4c22a8619d5eb2dd2e2 Mon Sep 17 00:00:00 2001 From: Jason Cooke Date: Wed, 17 Jul 2019 19:01:07 +1200 Subject: [PATCH 159/330] docs: fix typo (#837) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45e4158843..3ab22f90b3 100644 --- a/README.md +++ b/README.md @@ -849,7 +849,7 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime(); Normally, the entire duration of the work loop (`for (auto _ : state) {}`) is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. -That is possible, althought it is not recommended, since it has high overhead. +That is possible, although it is not recommended, since it has high overhead. ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { From 8e48105d465c586068dd8e248fe75a8971c6ba3a Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 22 Jul 2019 15:42:12 +0300 Subject: [PATCH 160/330] CMake; windows: link to lowercase 'shlwapi' - consistent with headers (#840) The filenames are consistently inconsistent in windows world, might have something to do with default file system being case-insensitive. While the native MinGW buils were fixed in 52613079824ac58d06c070aa9fbbb186a5859e2c that only addressed the headers, but not libraries. The problem remains when one tries to do a MinGW cross-build from case-sensitive filesystem. --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b47de6791c..eab1428e87 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,7 +44,7 @@ endif() # We need extra libraries on Windows if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - target_link_libraries(benchmark Shlwapi) + target_link_libraries(benchmark shlwapi) endif() # We need extra libraries on Solaris From 32a1e3972014c27b683945f45a2b4a37ab1040c1 Mon Sep 17 00:00:00 2001 From: Eric Backus <52862004+EricBackus@users.noreply.github.com> Date: Sat, 27 Jul 2019 09:02:31 -0700 Subject: [PATCH 161/330] Bugfix/wsl selftest fixes. Fixes #839 (#843) * Update AUTHORS and CONTRIBUTORS * Fix WSL self-test failures Some of the benchmark self-tests expect and check for a particular output format from the benchmark library. The numerical values must not be infinity or not-a-number, or the test will report an error. Some of the values are computed bytes-per-second or items-per-second values, so these require that the measured CPU time for the test to be non-zero. But the loop that is being measured was empty, so the measured CPU time for the loop was extremely small. On systems like Windows Subsystem for Linux (WSL) the timer doesn't have enough resolution to measure this, so the measured CPU time was zero. This fix just makes sure that these tests have something within the timing loop, so that the benchmark library will not decide that the loop takes zero CPU time. This makes these tests more robust, and in particular makes them pass on WSL. --- AUTHORS | 1 + CONTRIBUTORS | 1 + test/complexity_test.cc | 2 ++ test/reporter_output_test.cc | 6 ++++++ test/user_counters_tabular_test.cc | 2 ++ test/user_counters_test.cc | 10 ++++++++++ 6 files changed, 22 insertions(+) diff --git a/AUTHORS b/AUTHORS index 912cbbc13c..15418631c3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,6 +19,7 @@ David Coeurjolly Deniz Evrenci Dirac Research Dominik Czarnota +Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov diff --git a/CONTRIBUTORS b/CONTRIBUTORS index b680efc8c4..167165f469 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -35,6 +35,7 @@ David Coeurjolly Deniz Evrenci Dominic Hamon Dominik Czarnota +Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov diff --git a/test/complexity_test.cc b/test/complexity_test.cc index d4febbbc15..5681fdcf34 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -190,6 +190,8 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, void BM_ComplexityCaptureArgs(benchmark::State& state, int n) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } state.SetComplexityN(n); } diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index c8090d4aca..8486d590dc 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -90,6 +90,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } state.SetBytesProcessed(1); } @@ -117,6 +119,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } state.SetItemsProcessed(1); } @@ -262,6 +266,8 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } state.SetComplexityN(state.range(0)); } diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 099464ef99..18373c0aac 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -107,6 +107,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters.insert({ diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 0775bc01f7..4d308dfb45 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -64,6 +64,8 @@ int num_calls1 = 0; } void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -111,6 +113,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -228,6 +232,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -309,6 +315,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = @@ -394,6 +402,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; From ff7e2d45a5a016a7c54f6ce385efc59b3d975598 Mon Sep 17 00:00:00 2001 From: Boris Dergachov <42623189+froexilize@users.noreply.github.com> Date: Mon, 29 Jul 2019 23:59:06 +0300 Subject: [PATCH 162/330] README.md: Spelling fix (#845) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ab22f90b3..3c21937520 100644 --- a/README.md +++ b/README.md @@ -697,7 +697,7 @@ When you're compiling in C++11 mode or later you can use `insert()` with #### Counter Reporting -When using the console reporter, by default, user counters are are printed at +When using the console reporter, by default, user counters are printed at the end after the table, the same way as ``bytes_processed`` and ``items_processed``. This is best for cases in which there are few counters, or where there are only a couple of lines per benchmark. Here's an example of From 66482d538d6c6041b29ab6a95cf5b6d561fbd306 Mon Sep 17 00:00:00 2001 From: blackliner Date: Mon, 29 Jul 2019 22:59:25 +0200 Subject: [PATCH 163/330] README.md: corrected cmake commands (#846) * corrected cmake commands * Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c21937520..ea14d73b4a 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ $ git clone https://github.com/google/googletest.git benchmark/googletest $ mkdir build && cd build # Generate a Makefile with cmake. # Use cmake -G to generate a different file type. -$ cmake ../benchmark +$ cmake ../ # Build the library. $ make ``` From 140db8a22901f666577b7516febf184e9764f4e2 Mon Sep 17 00:00:00 2001 From: Xinye Tao Date: Tue, 30 Jul 2019 14:03:15 +0800 Subject: [PATCH 164/330] fix typo in README (#844) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea14d73b4a..0efe48fda3 100644 --- a/README.md +++ b/README.md @@ -1088,7 +1088,7 @@ static void BM_test(benchmark::State& state) { state.SkipWithError("Resource is not good!"); // KeepRunning() loop will not be entered. } - for (state.KeepRunning()) { + while (state.KeepRunning()) { auto data = resource.read_data(); if (!resource.good()) { state.SkipWithError("Failed to read data!"); From 140fc22ab22c770b44228ec6b38c745fc610269d Mon Sep 17 00:00:00 2001 From: LesnyRumcajs Date: Tue, 6 Aug 2019 12:36:36 +0200 Subject: [PATCH 165/330] Corrected the installation procedure (#849) * Corrected the installation procedure Now it can be put into a script. * Updated the file tree Necessary after installation instruction change --- README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0efe48fda3..f576973236 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,15 @@ versions of build tools._ $ git clone https://github.com/google/benchmark.git # Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory. $ git clone https://github.com/google/googletest.git benchmark/googletest +# Go to the library root directory +$ cd benchmark # Make a build directory to place the build output. $ mkdir build && cd build # Generate a Makefile with cmake. # Use cmake -G to generate a different file type. $ cmake ../ # Build the library. +# Use make -j to speed up the build process, e.g. make -j8 . $ make ``` This builds the `benchmark` and `benchmark_main` libraries and tests. @@ -78,12 +81,12 @@ On a unix system, the build directory should now look something like this: ``` /benchmark -/build - /src - /libbenchmark.a - /libbenchmark_main.a - /test - ... + /build + /src + /libbenchmark.a + /libbenchmark_main.a + /test + ... ``` Next, you can run the tests to check the build. From c408461983dd3adf49d450d7db926fc46f1d99a0 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 7 Aug 2019 15:55:40 -0400 Subject: [PATCH 166/330] Disable deprecated warnings when touching CSVReporter internally. The CSVReporter is deprecated, but we still need to reference it in a few places. To avoid breaking the build when warnings are errors, we need to disable the warning when we do so. --- src/benchmark.cc | 12 ++++++++++++ test/output_test_helper.cc | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/benchmark.cc b/src/benchmark.cc index 29bfa3512f..67c9744a84 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -289,6 +289,13 @@ void RunBenchmarks(const std::vector& benchmarks, flushStreams(file_reporter); } +// Disable deprecated warnings temporarily because we need to reference +// CSVReporter but don't want to trigger -Werror=-Wdeprecated +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#endif + std::unique_ptr CreateReporter( std::string const& name, ConsoleReporter::OutputOptions output_opts) { typedef std::unique_ptr PtrType; @@ -304,6 +311,11 @@ std::unique_ptr CreateReporter( } } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + } // end namespace bool IsZero(double n) { diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 5dc951d2bc..bdb34c8796 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -373,6 +373,12 @@ int SetSubstitutions( return 0; } +// Disable deprecated warnings temporarily because we need to reference +// CSVReporter but don't want to trigger -Werror=-Wdeprecated +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#endif void RunOutputTests(int argc, char* argv[]) { using internal::GetTestCaseList; benchmark::Initialize(&argc, argv); @@ -431,6 +437,10 @@ void RunOutputTests(int argc, char* argv[]) { internal::GetResultsChecker().CheckResults(csv.out_stream); } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + int SubstrCnt(const std::string& haystack, const std::string& pat) { if (pat.length() == 0) return 0; int count = 0; From 7d97a057e16597e8020d0aca110480fe82c9ca67 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 12 Aug 2019 17:47:46 +0300 Subject: [PATCH 167/330] Custom user counters: add invert modifier. (#850) While current counters can e.g. answer the question "how many items is processed per second", it is impossible to get it to tell "how many seconds it takes to process a single item". The solution is to add a yet another modifier `kInvert`, that is *always* considered last, which simply inverts the answer. Fixes #781, #830, #848. --- README.md | 12 +++-- include/benchmark/benchmark.h | 5 ++- src/console_reporter.cc | 22 +++++----- src/counter.cc | 4 ++ test/user_counters_test.cc | 83 +++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f576973236..ebbbf158de 100644 --- a/README.md +++ b/README.md @@ -662,9 +662,9 @@ the resulting sum is the value which will be shown for the benchmark. The `Counter` constructor accepts three parameters: the value as a `double` ; a bit flag which allows you to show counters as rates, and/or as per-thread -iteration, and/or as per-thread averages, and/or iteration invariants; -and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, -`benchmark::Counter::OneK::kIs1000`), or 1024 +iteration, and/or as per-thread averages, and/or iteration invariants, +and/or finally inverting the result; and a flag specifying the 'unit' - i.e. +is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 (`benchmark::Counter::OneK::kIs1024`)? ```c++ @@ -673,8 +673,14 @@ and a flag specifying the 'unit' - i.e. is 1k a 1000 (default, // Set the counter as a rate. It will be presented divided // by the duration of the benchmark. + // Meaning: per one second, how many 'foo's are processed? state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); + // Set the counter as a rate. It will be presented divided + // by the duration of the benchmark, and the result inverted. + // Meaning: how many seconds it takes to process one 'foo'? + state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert); + // Set the counter as a thread-average quantity. It will // be presented divided by the number of threads. state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 856605f25b..144e212e1b 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -368,7 +368,10 @@ class Counter { // It will be presented divided by the number of iterations. kAvgIterations = 1U << 3U, // Mark the counter as a iteration-average rate. See above. - kAvgIterationsRate = kIsRate | kAvgIterations + kAvgIterationsRate = kIsRate | kAvgIterations, + + // In the end, invert the result. This is always done last! + kInvert = 1U << 31U }; enum OneK { diff --git a/src/console_reporter.cc b/src/console_reporter.cc index cc8ae276f6..6fd764525e 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -12,21 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" -#include "complexity.h" -#include "counter.h" - #include #include #include +#include #include #include #include #include +#include "benchmark/benchmark.h" #include "check.h" #include "colorprint.h" #include "commandlineflags.h" +#include "complexity.h" +#include "counter.h" #include "internal_macros.h" #include "string_util.h" #include "timers.h" @@ -156,16 +156,14 @@ void ConsoleReporter::PrintRunData(const Run& result) { const std::size_t cNameLen = std::max(std::string::size_type(10), c.first.length()); auto const& s = HumanReadableNumber(c.second.value, c.second.oneK); + const char* unit = ""; + if (c.second.flags & Counter::kIsRate) + unit = (c.second.flags & Counter::kInvert) ? "s" : "/s"; if (output_options_ & OO_Tabular) { - if (c.second.flags & Counter::kIsRate) { - printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str()); - } else { - printer(Out, COLOR_DEFAULT, " %*s", cNameLen, s.c_str()); - } - } else { - const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : ""; - printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), + printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(), unit); + } else { + printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit); } } diff --git a/src/counter.cc b/src/counter.cc index c248ea110b..cf5b78ee3a 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -32,6 +32,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, if (c.flags & Counter::kAvgIterations) { v /= iterations; } + + if (c.flags & Counter::kInvert) { // Invert is *always* last. + v = 1.0 / v; + } return v; } diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 4d308dfb45..5699f4f5e1 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -148,6 +148,89 @@ void CheckRate(Results const& e) { } CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); +// ========================================================================= // +// ----------------------- Inverted Counters Output ------------------------ // +// ========================================================================= // + +void BM_Invert(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; + state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert}; +} +BENCHMARK(BM_Invert); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, + {"\"run_name\": \"BM_Invert\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Invert\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvert(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 10000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001); +} +CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); + +// ========================================================================= // +// ------------------------- InvertedRate Counters Output +// -------------------------- // +// ========================================================================= // + +void BM_Counters_InvertedRate(benchmark::State& state) { + for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + } + namespace bm = benchmark; + state.counters["foo"] = + bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert}; + state.counters["bar"] = + bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert}; +} +BENCHMARK(BM_Counters_InvertedRate); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " + "bar=%hrfloats foo=%hrfloats$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_InvertedRate\",$"}, + {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_InvertedRate\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckInvertedRate(Results const& e) { + double t = e.DurationCPUTime(); // this (and not real time) is the time used + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate); + // ========================================================================= // // ------------------------- Thread Counters Output ------------------------ // // ========================================================================= // From 3523f11d36caf2ffe4acffe2e9fb15775cec8112 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 21 Aug 2019 15:14:27 +0300 Subject: [PATCH 168/330] Update README.md: fix MS VS version requirement VS2013 is unsupported since https://github.com/google/benchmark/pull/691 / https://github.com/google/benchmark/commit/eb8cbec0776455463274ea9947ab0ecfe0f768fe but i forgot to update docs. References: * https://github.com/google/benchmark/issues/689 * https://github.com/google/benchmark/pull/691 * https://github.com/google/googletest/pull/1815 * https://github.com/google/benchmark/issues/853 * https://github.com/google/benchmark/pull/854 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebbbf158de..e9fbe130e3 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The following minimum versions are required to build the library: * GCC 4.8 * Clang 3.4 -* Visual Studio 2013 +* Visual Studio 14 2015 * Intel 2015 Update 1 ## Installation From ffadb65d3a17452b2bd08951d1bce6b0f6bf65d9 Mon Sep 17 00:00:00 2001 From: Sayan Bhattacharjee Date: Wed, 21 Aug 2019 22:21:31 +0530 Subject: [PATCH 169/330] Documentation of basic use of DenseRange. (#855) Documentation of basic use of `DenseRange` was added to README.md. --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 16 ++++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/AUTHORS b/AUTHORS index 15418631c3..c5e5c0c497 100644 --- a/AUTHORS +++ b/AUTHORS @@ -44,6 +44,7 @@ Ori Livneh Paul Redmond Radoslav Yovchev Roman Lebedev +Sayan Bhattacharjee Shuo Chen Steinar H. Gunderson Stripe, Inc. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 167165f469..5be6dc48cf 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -65,6 +65,7 @@ Raul Marin Ray Glover Robert Guo Roman Lebedev +Sayan Bhattacharjee Shuo Chen Tobias Ulvgård Tom Madams diff --git a/README.md b/README.md index e9fbe130e3..eb9374c256 100644 --- a/README.md +++ b/README.md @@ -422,6 +422,22 @@ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. +The preceding code shows a method of defining a sparse range. The following +example shows a method of defining a dense range. It is then used to benchmark +the performance of `std::vector` initialization for uniformly increasing sizes. + +```c++ +static void BM_DenseRange(benchmark::State& state) { + for(auto _ : state) { + std::vector v(state.range(0), state.range(0)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128); +``` +Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ]. + You might have a benchmark that depends on two or more inputs. For example, the following code defines a family of benchmarks for measuring the speed of set insertion. From 67853d3ed844fb1a2f9d47886a6820db448703b0 Mon Sep 17 00:00:00 2001 From: Sayan Bhattacharjee Date: Thu, 22 Aug 2019 02:23:15 +0530 Subject: [PATCH 170/330] Add .gitignore rule to ignore temporary .swp backup files created by vim. (#859) - Addresses : #858 - Rule `*.swp` is added to `.gitignore` to ensure that the vim temporary `.swp` backup files are ignored and they don't pollute the results of `git status -u`. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 806d04c6b3..a7716e3d5d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ !/cmake/*.cmake !/test/AssemblyTests.cmake *~ +*.swp *.pyc __pycache__ From 7ee72863fdb1ccb2af5a011250b56af3f49b7511 Mon Sep 17 00:00:00 2001 From: Sayan Bhattacharjee Date: Thu, 22 Aug 2019 02:42:03 +0530 Subject: [PATCH 171/330] Remove unused `doc` argument from `DEFINE_` macros. (#857) - Adresses : #856 - The unused `doc` argument was removed from the `DEFINE_` macros in `commandlineflags.h` - Converted all the previous `doc` strings passed to the `DEFINE_` macros to multiline comments. --- src/benchmark.cc | 114 +++++++++++++++++++---------------------- src/commandlineflags.h | 11 ++-- 2 files changed, 59 insertions(+), 66 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 67c9744a84..9af0701399 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -51,66 +51,60 @@ #include "thread_manager.h" #include "thread_timer.h" -DEFINE_bool(benchmark_list_tests, false, - "Print a list of benchmarks. This option overrides all other " - "options."); - -DEFINE_string(benchmark_filter, ".", - "A regular expression that specifies the set of benchmarks " - "to execute. If this flag is empty, or if this flag is the " - "string \"all\", all benchmarks linked into the binary are " - "run."); - -DEFINE_double(benchmark_min_time, 0.5, - "Minimum number of seconds we should run benchmark before " - "results are considered significant. For cpu-time based " - "tests, this is the lower bound on the total cpu time " - "used by all threads that make up the test. For real-time " - "based tests, this is the lower bound on the elapsed time " - "of the benchmark execution, regardless of number of " - "threads."); - -DEFINE_int32(benchmark_repetitions, 1, - "The number of runs of each benchmark. If greater than 1, the " - "mean and standard deviation of the runs will be reported."); - -DEFINE_bool( - benchmark_report_aggregates_only, false, - "Report the result of each benchmark repetitions. When 'true' is specified " - "only the mean, standard deviation, and other statistics are reported for " - "repeated benchmarks. Affects all reporters."); - -DEFINE_bool( - benchmark_display_aggregates_only, false, - "Display the result of each benchmark repetitions. When 'true' is " - "specified only the mean, standard deviation, and other statistics are " - "displayed for repeated benchmarks. Unlike " - "benchmark_report_aggregates_only, only affects the display reporter, but " - "*NOT* file reporter, which will still contain all the output."); - -DEFINE_string(benchmark_format, "console", - "The format to use for console output. Valid values are " - "'console', 'json', or 'csv'."); - -DEFINE_string(benchmark_out_format, "json", - "The format to use for file output. Valid values are " - "'console', 'json', or 'csv'."); - -DEFINE_string(benchmark_out, "", "The file to write additional output to"); - -DEFINE_string(benchmark_color, "auto", - "Whether to use colors in the output. Valid values: " - "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use " - "colors if the output is being sent to a terminal and the TERM " - "environment variable is set to a terminal type that supports " - "colors."); - -DEFINE_bool(benchmark_counters_tabular, false, - "Whether to use tabular format when printing user counters to " - "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0." - "Defaults to false."); - -DEFINE_int32(v, 0, "The level of verbose logging to output"); +// Print a list of benchmarks. This option overrides all other options. +DEFINE_bool(benchmark_list_tests, false); + +// A regular expression that specifies the set of benchmarks to execute. If +// this flag is empty, or if this flag is the string \"all\", all benchmarks +// linked into the binary are run. +DEFINE_string(benchmark_filter, "."); + +// Minimum number of seconds we should run benchmark before results are +// considered significant. For cpu-time based tests, this is the lower bound +// on the total cpu time used by all threads that make up the test. For +// real-time based tests, this is the lower bound on the elapsed time of the +// benchmark execution, regardless of number of threads. +DEFINE_double(benchmark_min_time, 0.5); + +// The number of runs of each benchmark. If greater than 1, the mean and +// standard deviation of the runs will be reported. +DEFINE_int32(benchmark_repetitions, 1); + +// Report the result of each benchmark repetitions. When 'true' is specified +// only the mean, standard deviation, and other statistics are reported for +// repeated benchmarks. Affects all reporters. +DEFINE_bool( benchmark_report_aggregates_only, false); + +// Display the result of each benchmark repetitions. When 'true' is specified +// only the mean, standard deviation, and other statistics are displayed for +// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects +// the display reporter, but *NOT* file reporter, which will still contain +// all the output. +DEFINE_bool( benchmark_display_aggregates_only, false); + +// The format to use for console output. +// Valid values are 'console', 'json', or 'csv'. +DEFINE_string(benchmark_format, "console"); + +// The format to use for file output. +// Valid values are 'console', 'json', or 'csv'. +DEFINE_string(benchmark_out_format, "json"); + +// The file to write additional output to. +DEFINE_string(benchmark_out, ""); + +// Whether to use colors in the output. Valid values: +// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if +// the output is being sent to a terminal and the TERM environment variable is +// set to a terminal type that supports colors. +DEFINE_string(benchmark_color, "auto"); + +// Whether to use tabular format when printing user counters to the console. +// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. +DEFINE_bool(benchmark_counters_tabular, false); + +// The level of verbose logging to output +DEFINE_int32(v, 0); namespace benchmark { diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 5eaea82a59..afe52389b1 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -15,12 +15,11 @@ #define DECLARE_string(name) extern std::string FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val) -#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val) -#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val) -#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val) -#define DEFINE_string(name, default_val, doc) \ - std::string FLAG(name) = (default_val) +#define DEFINE_bool(name, default_val) bool FLAG(name) = (default_val) +#define DEFINE_int32(name, default_val) int32_t FLAG(name) = (default_val) +#define DEFINE_int64(name, default_val) int64_t FLAG(name) = (default_val) +#define DEFINE_double(name, default_val) double FLAG(name) = (default_val) +#define DEFINE_string(name, default_val) std::string FLAG(name) = (default_val) namespace benchmark { // Parses a bool/Int32/string from the environment variable From ef7d51c8ebf6fdc10687b4e9eaa48c72461e771d Mon Sep 17 00:00:00 2001 From: "Attila M. Szilagyi" Date: Sun, 15 Sep 2019 10:25:32 -0700 Subject: [PATCH 172/330] Allow setting GOOGLETEST_PATH cmake argument. Fixes #867 (#868) In `cmake/GoogleTest.cmake`, GOOGLETEST_PATH is default-initialized, but that init forgot to account for the fact that the patch is explicitly supposed to be user-configurable. By passing `CACHE` to `set()` we avoid that error. --- cmake/GoogleTest.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake index fb7c6be25e..dd611fc875 100644 --- a/cmake/GoogleTest.cmake +++ b/cmake/GoogleTest.cmake @@ -2,7 +2,7 @@ set(GOOGLETEST_PREFIX "${benchmark_BINARY_DIR}/third_party/googletest") configure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY) -set(GOOGLETEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/googletest") # Mind the quotes +set(GOOGLETEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/googletest" CACHE PATH "") # Mind the quotes execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH=${GOOGLETEST_PATH} . RESULT_VARIABLE result From bf4f2ea0bd1180b34718ac26eb79b170a4f6290e Mon Sep 17 00:00:00 2001 From: sharpe5 Date: Mon, 16 Sep 2019 09:05:05 +0100 Subject: [PATCH 173/330] Addresses issue #634. (#866) * Update with instructions to build under Visual Studio Fixes Issue #634. I spent 3 days trying to build this library under Visual Studio 2017, only to discover on has to link to `Shlwapi.lib`. Became so frustrated with the docs that I added full build instructions for Visual Studio 2015, 2017 and Intel Comiler 2015 and 2019. * Update headings --- README.md | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index eb9374c256..c9aea0f538 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ The following minimum versions are required to build the library: * Visual Studio 14 2015 * Intel 2015 Update 1 +See [Platform-Specific Build Instructions](#platform-specific-build-instructions). + ## Installation This describes the installation process using cmake. As pre-requisites, you'll @@ -190,7 +192,9 @@ Alternatively, link against the `benchmark_main` library and remove The compiled executable will run all benchmarks by default. Pass the `--help` flag for option information or see the guide below. -### Platform-specific instructions +## Platform Specific Build Instructions + +### Building with GCC When the library is built using GCC it is necessary to link with the pthread library due to how GCC implements `std::thread`. Failing to link to pthread will @@ -200,8 +204,34 @@ can link to pthread by adding `-pthread` to your linker command. Note, you can also use `-lpthread`, but there are potential issues with ordering of command line parameters if you use that. -If you're running benchmarks on Windows, the shlwapi library (`-lshlwapi`) is -also required. +### Building with Visual Studio 2015 or 2017 + +The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following: + +``` +// Alternatively, can add libraries using linker options. +#ifdef _WIN32 +#pragma comment ( lib, "Shlwapi.lib" ) +#ifdef _DEBUG +#pragma comment ( lib, "benchmarkd.lib" ) +#else +#pragma comment ( lib, "benchmark.lib" ) +#endif +#endif +``` + +Can also use the graphical version of CMake: +* Open `CMake GUI`. +* Under `Where to build the binaries`, same path as source plus `build`. +* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`. +* Click `Configure`, `Generate`, `Open Project`. +* If build fails, try deleting entire directory and starting again, or unticking options to build less. + +### Building with Intel 2015 Update 1 or Intel System Studio Update 4 + +See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel. + +### Building on Solaris If you're running benchmarks on solaris, you'll want the kstat library linked in too (`-lkstat`). From d2fc7fe6591393c2672f3ba011304b778e71c020 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Fri, 20 Sep 2019 02:25:31 -0700 Subject: [PATCH 174/330] Guard ASSERT_THROWS checks with BENCHMARK_HAS_NO_EXCEPTIONS (#874) * Guard ASSERT_THROWS checks with BENCHMARK_HAS_NO_EXCEPTIONS This allows the test be run with exceptions turned off * Add myself to CONTRIBUTORS I don't need to be added to AUTHORS, as I am a Google employee --- CONTRIBUTORS | 1 + test/string_util_gtest.cc | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 5be6dc48cf..e44f6f672e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -41,6 +41,7 @@ Eugene Zhuk Evgeny Safronov Federico Ficarelli Felix Homann +Geoffrey Martin-Noble Hannes Hauswedell Ismael Jimenez Martinez Jern-Kuan Leong diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 2c5d073f61..01bf155d8c 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -3,6 +3,7 @@ //===---------------------------------------------------------------------===// #include "../src/string_util.h" +#include "../src/internal_macros.h" #include "gtest/gtest.h" namespace { @@ -60,9 +61,11 @@ TEST(StringUtilTest, stoul) { EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); EXPECT_EQ(4ul, pos); } +#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } +#endif } TEST(StringUtilTest, stoi) { @@ -106,9 +109,11 @@ TEST(StringUtilTest, stoi) { EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); EXPECT_EQ(4ul, pos); } +#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } +#endif } TEST(StringUtilTest, stod) { @@ -138,9 +143,11 @@ TEST(StringUtilTest, stod) { EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); EXPECT_EQ(8ul, pos); } +#ifndef BENCHMARK_HAS_NO_EXCEPTIONS { ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } +#endif } } // end namespace From e7e3d976ef7d89ffc6bd6a53a6ea13ec35bb411d Mon Sep 17 00:00:00 2001 From: Colin Braley Date: Sat, 21 Sep 2019 13:55:05 -0700 Subject: [PATCH 175/330] Add missing parenthesis to code sample, and fix spacing. (#877) * Fix missing paren in README code sample, and fix code spacing. * Add Colin Braley to AUTHORS and CONTRIBUTORS. --- AUTHORS | 1 + CONTRIBUTORS | 3 ++- README.md | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index c5e5c0c497..35c4c8cee9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -14,6 +14,7 @@ Andriy Berestovskyy Arne Beer Carto Christopher Seymour +Colin Braley Daniel Harvey David Coeurjolly Deniz Evrenci diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e44f6f672e..6b64a00628 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -29,6 +29,7 @@ Arne Beer Billy Robert O'Neal III Chris Kennelly Christopher Seymour +Colin Braley Cyrille Faucheux Daniel Harvey David Coeurjolly @@ -50,8 +51,8 @@ Joao Paulo Magalhaes John Millikin Jussi Knuuttila Kai Wolf -Kishan Kumar Kaito Udagawa +Kishan Kumar Lei Xu Matt Clarkson Maxim Vafin diff --git a/README.md b/README.md index c9aea0f538..7eaa02a75c 100644 --- a/README.md +++ b/README.md @@ -873,7 +873,7 @@ static void MyMain(int size) { static void BM_OpenMP(benchmark::State& state) { for (auto _ : state) - MyMain(state.range(0); + MyMain(state.range(0)); } // Measure the time spent by the main thread, use it to decide for how long to @@ -950,7 +950,7 @@ static void BM_ManualTiming(benchmark::State& state) { auto start = std::chrono::high_resolution_clock::now(); // Simulate some useful workload with a sleep std::this_thread::sleep_for(sleep_duration); - auto end = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::high_resolution_clock::now(); auto elapsed_seconds = std::chrono::duration_cast>( From 7411874d9563b18c56f8a81e02e77c6ffc5c3851 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Mon, 23 Sep 2019 02:38:34 -0700 Subject: [PATCH 176/330] Define HOST_NAME_MAX for NaCl and RTEMS (#876) These OS's don't always have HOST_NAME_MAX defined, resulting in build errors. A few related changes as well: * Only define HOST_NAME_MAX if it's not already defined. There are some cases where this is already defined, e.g. with NaCl if __USE_POSIX is set. To avoid all of these, only define it if it's not already defined. * Default HOST_NAME_MAX to 64 and issue a #warning. Having the wrong max length is pretty harmless. The name just ends up getting truncated and this is only for printing debug info. Because we're constructing a std::string from a char[] (so defined length), we don't need to worry about gethostname's undefined behavior for whether the truncation is null-terminated when the hostname doesn't fit in HOST_NAME_MAX. Of course, this doesn't help people who have -Werror set, since they'll still get a warning. --- src/sysinfo.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 28126470ba..b5f99c7de5 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -429,11 +429,20 @@ std::string GetSystemName() { #endif return str; #else // defined(BENCHMARK_OS_WINDOWS) +#ifndef HOST_NAME_MAX #ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 +#elif defined(BENCHMARK_OS_NACL) +#define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_QNX) #define HOST_NAME_MAX 154 +#elif defined(BENCHMARK_OS_RTEMS) +#define HOST_NAME_MAX 256 +#else +#warning "HOST_NAME_MAX not defined. using 64" +#define HOST_NAME_MAX 64 #endif +#endif // def HOST_NAME_MAX char hostname[HOST_NAME_MAX]; int retVal = gethostname(hostname, HOST_NAME_MAX); if (retVal != 0) return std::string(""); From b874e72208b6e21b62287942e5e3b11f6630107f Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Mon, 23 Sep 2019 02:53:09 -0700 Subject: [PATCH 177/330] Guard definition of __STDC_FORMAT_MACROS in ifndef (#875) This macro is sometimes already defined and redefining it results in build errors. --- src/benchmark_register.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 6696c382b8..cca39b2215 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -34,7 +34,9 @@ #include #include +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include #include "benchmark/benchmark.h" From b8bce0c7ed4e02426fa4de21fec385fbaa84a7b2 Mon Sep 17 00:00:00 2001 From: Chunsheng Pei Date: Fri, 27 Sep 2019 06:09:23 -0500 Subject: [PATCH 178/330] fixed the param order in g++ command and fixed the path for -L (#879) * fixed the param order in g++ command and fixed the path for -L * reorder the parameters for g++ command --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7eaa02a75c..a8aa276f6c 100644 --- a/README.md +++ b/README.md @@ -182,8 +182,8 @@ library will be under the build directory you created. ```bash # Example on linux after running the build steps above. Assumes the # `benchmark` and `build` directories are under the current directory. -$ g++ -std=c++11 -isystem benchmark/include -Lbuild/src -lpthread \ - -lbenchmark mybenchmark.cc -o mybenchmark +$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \ + -Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark ``` Alternatively, link against the `benchmark_main` library and remove From f4f5dba46bdbde0e95d736cca124025745bcd7b6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 4 Oct 2019 04:50:53 -0400 Subject: [PATCH 179/330] Cache RUN_${FEATURE} variable in CXXFeatureCheck.cmake (#886) When cross-compiling, this variable was not set on the second run of CMake, resulting in the next check failing even though it shouldn't be run in the first place. Fix this by caching the variable. --- cmake/CXXFeatureCheck.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index 99b56dd623..059d510dd9 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -37,9 +37,9 @@ function(cxx_feature_check FILE) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") - set(RUN_${FEATURE} 0) + set(RUN_${FEATURE} 0 CACHE INTERNAL "") else() - set(RUN_${FEATURE} 1) + set(RUN_${FEATURE} 1 CACHE INTERNAL "") endif() else() message(STATUS "Performing Test ${FEATURE}") From 309de5988eb949a27e077a24a1d83c0687d10d57 Mon Sep 17 00:00:00 2001 From: Paul Wankadia Date: Tue, 8 Oct 2019 21:09:51 +1100 Subject: [PATCH 180/330] Switch to Starlark for C++ rules. (#887) While I'm here, format all of the files that I touched. --- BUILD.bazel | 2 ++ WORKSPACE | 12 +++++++++--- test/BUILD | 48 ++++++++++++++++++++++++++++-------------------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 6ee69f2907..d97a019bee 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -8,6 +8,8 @@ config_setting( visibility = [":__subpackages__"], ) +load("@rules_cc//cc:defs.bzl", "cc_library") + cc_library( name = "benchmark", srcs = glob( diff --git a/WORKSPACE b/WORKSPACE index 9a75f968d9..8df248a4c0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,7 +3,13 @@ workspace(name = "com_github_google_benchmark") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( - name = "com_google_googletest", - urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], - strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", + name = "rules_cc", + strip_prefix = "rules_cc-a508235df92e71d537fcbae0c7c952ea6957a912", + urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"], +) + +http_archive( + name = "com_google_googletest", + strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", + urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], ) diff --git a/test/BUILD b/test/BUILD index 3f174c486f..9bb8cb02a0 100644 --- a/test/BUILD +++ b/test/BUILD @@ -5,7 +5,7 @@ TEST_COPTS = [ "-Wall", "-Wextra", "-Wshadow", -# "-Wshorten-64-to-32", + # "-Wshorten-64-to-32", "-Wfloat-equal", "-fstrict-aliasing", ] @@ -16,13 +16,14 @@ PER_SRC_COPTS = ({ "donotoptimize_test.cc": ["-O3"], }) - TEST_ARGS = ["--benchmark_min_time=0.01"] PER_SRC_TEST_ARGS = ({ "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], }) +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + cc_library( name = "output_test_helper", testonly = 1, @@ -36,24 +37,31 @@ cc_library( ) [ - cc_test( - name = test_src[:-len(".cc")], - size = "small", - srcs = [test_src], - args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), - copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []), - deps = [ - ":output_test_helper", - "//:benchmark", - "//:benchmark_internal_headers", - "@com_google_googletest//:gtest", - ] + ( - ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else [] - ), - # FIXME: Add support for assembly tests to bazel. - # See Issue #556 - # https://github.com/google/benchmark/issues/556 - ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc", "link_main_test.cc"]) + cc_test( + name = test_src[:-len(".cc")], + size = "small", + srcs = [test_src], + args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []), + copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []), + deps = [ + ":output_test_helper", + "//:benchmark", + "//:benchmark_internal_headers", + "@com_google_googletest//:gtest", + ] + ( + ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else [] + ), + # FIXME: Add support for assembly tests to bazel. + # See Issue #556 + # https://github.com/google/benchmark/issues/556 + ) + for test_src in glob( + ["*test.cc"], + exclude = [ + "*_assembly_test.cc", + "link_main_test.cc", + ], + ) ] cc_test( From bc200ed8eeb179754d483351dc526143b1f9043b Mon Sep 17 00:00:00 2001 From: Martin Blanchard Date: Wed, 23 Oct 2019 10:07:08 +0200 Subject: [PATCH 181/330] Read options from environment (#881) (#883) Initialize option flags from environment variables values if they are defined, eg. `BENCHMARK_OUT=` for `--benchmark_out=`. Command line flag value always prevails. Fixes https://github.com/google/benchmark/issues/881. --- src/benchmark.cc | 9 ++- src/commandlineflags.cc | 68 ++++++++++-------- src/commandlineflags.h | 51 ++++++++++--- test/commandlineflags_gtest.cc | 127 ++++++++++++++++++++++++++++++++- 4 files changed, 207 insertions(+), 48 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 9af0701399..07942eba65 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -73,14 +73,14 @@ DEFINE_int32(benchmark_repetitions, 1); // Report the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are reported for // repeated benchmarks. Affects all reporters. -DEFINE_bool( benchmark_report_aggregates_only, false); +DEFINE_bool(benchmark_report_aggregates_only, false); // Display the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are displayed for // repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects // the display reporter, but *NOT* file reporter, which will still contain // all the output. -DEFINE_bool( benchmark_display_aggregates_only, false); +DEFINE_bool(benchmark_display_aggregates_only, false); // The format to use for console output. // Valid values are 'console', 'json', or 'csv'. @@ -142,7 +142,7 @@ State::State(IterationCount max_iters, const std::vector& ranges, // which must be suppressed. #if defined(__INTEL_COMPILER) #pragma warning push -#pragma warning(disable:1875) +#pragma warning(disable : 1875) #elif defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -309,7 +309,6 @@ std::unique_ptr CreateReporter( #pragma GCC diagnostic pop #endif - } // end namespace bool IsZero(double n) { @@ -318,7 +317,7 @@ bool IsZero(double n) { ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { int output_opts = ConsoleReporter::OO_Defaults; - auto is_benchmark_color = [force_no_color] () -> bool { + auto is_benchmark_color = [force_no_color]() -> bool { if (force_no_color) { return false; } diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 6bd65c5ae7..4e60f0bd06 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -14,6 +14,7 @@ #include "commandlineflags.h" +#include #include #include #include @@ -92,44 +93,40 @@ static std::string FlagToEnvVar(const char* flag) { } // namespace -// Reads and returns the Boolean environment variable corresponding to -// the given flag; if it's not set, returns default_value. -// -// The value is considered true iff it's not "0". -bool BoolFromEnv(const char* flag, bool default_value) { +bool BoolFromEnv(const char* flag, bool default_val) { const std::string env_var = FlagToEnvVar(flag); - const char* const string_value = getenv(env_var.c_str()); - return string_value == nullptr ? default_value - : strcmp(string_value, "0") != 0; + const char* const value_str = getenv(env_var.c_str()); + return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); } -// Reads and returns a 32-bit integer stored in the environment -// variable corresponding to the given flag; if it isn't set or -// doesn't represent a valid 32-bit integer, returns default_value. -int32_t Int32FromEnv(const char* flag, int32_t default_value) { +int32_t Int32FromEnv(const char* flag, int32_t default_val) { const std::string env_var = FlagToEnvVar(flag); - const char* const string_value = getenv(env_var.c_str()); - if (string_value == nullptr) { - // The environment variable is not set. - return default_value; + const char* const value_str = getenv(env_var.c_str()); + int32_t value = default_val; + if (value_str == nullptr || + !ParseInt32(std::string("Environment variable ") + env_var, value_str, + &value)) { + return default_val; } + return value; +} - int32_t result = default_value; - if (!ParseInt32(std::string("Environment variable ") + env_var, string_value, - &result)) { - std::cout << "The default value " << default_value << " is used.\n"; - return default_value; +double DoubleFromEnv(const char* flag, double default_val) { + const std::string env_var = FlagToEnvVar(flag); + const char* const value_str = getenv(env_var.c_str()); + double value = default_val; + if (value_str == nullptr || + !ParseDouble(std::string("Environment variable ") + env_var, value_str, + &value)) { + return default_val; } - - return result; + return value; } -// Reads and returns the string environment variable corresponding to -// the given flag; if it's not set, returns default_value. -const char* StringFromEnv(const char* flag, const char* default_value) { +const char* StringFromEnv(const char* flag, const char* default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); - return value == nullptr ? default_value : value; + return value == nullptr ? default_val : value; } // Parses a string as a command line flag. The string should have @@ -214,9 +211,18 @@ bool IsFlag(const char* str, const char* flag) { } bool IsTruthyFlagValue(const std::string& value) { - if (value.empty()) return true; - char ch = value[0]; - return isalnum(ch) && - !(ch == '0' || ch == 'f' || ch == 'F' || ch == 'n' || ch == 'N'); + if (value.size() == 1) { + char v = value[0]; + return isalnum(v) && + !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); + } else if (!value.empty()) { + std::string value_lower(value); + std::transform(value_lower.begin(), value_lower.end(), + value_lower.begin(), ::tolower); + return !(value_lower == "false" || value_lower == "no" || + value_lower == "off"); + } else + return true; } + } // end namespace benchmark diff --git a/src/commandlineflags.h b/src/commandlineflags.h index afe52389b1..3a1f6a8dbc 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -10,22 +10,51 @@ // Macros for declaring flags. #define DECLARE_bool(name) extern bool FLAG(name) #define DECLARE_int32(name) extern int32_t FLAG(name) -#define DECLARE_int64(name) extern int64_t FLAG(name) #define DECLARE_double(name) extern double FLAG(name) #define DECLARE_string(name) extern std::string FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val) bool FLAG(name) = (default_val) -#define DEFINE_int32(name, default_val) int32_t FLAG(name) = (default_val) -#define DEFINE_int64(name, default_val) int64_t FLAG(name) = (default_val) -#define DEFINE_double(name, default_val) double FLAG(name) = (default_val) -#define DEFINE_string(name, default_val) std::string FLAG(name) = (default_val) +#define DEFINE_bool(name, default_val) \ + bool FLAG(name) = \ + benchmark::BoolFromEnv(#name, default_val) +#define DEFINE_int32(name, default_val) \ + int32_t FLAG(name) = \ + benchmark::Int32FromEnv(#name, default_val) +#define DEFINE_double(name, default_val) \ + double FLAG(name) = \ + benchmark::DoubleFromEnv(#name, default_val) +#define DEFINE_string(name, default_val) \ + std::string FLAG(name) = \ + benchmark::StringFromEnv(#name, default_val) namespace benchmark { -// Parses a bool/Int32/string from the environment variable -// corresponding to the given Google Test flag. + +// Parses a bool from the environment variable +// corresponding to the given flag. +// +// If the variable exists, returns IsTruthyFlagValue() value; if not, +// returns the given default value. bool BoolFromEnv(const char* flag, bool default_val); + +// Parses an Int32 from the environment variable +// corresponding to the given flag. +// +// If the variable exists, returns ParseInt32() value; if not, returns +// the given default value. int32_t Int32FromEnv(const char* flag, int32_t default_val); + +// Parses an Double from the environment variable +// corresponding to the given flag. +// +// If the variable exists, returns ParseDouble(); if not, returns +// the given default value. +double DoubleFromEnv(const char* flag, double default_val); + +// Parses a string from the environment variable +// corresponding to the given flag. +// +// If variable exists, returns its value; if not, returns +// the given default value. const char* StringFromEnv(const char* flag, const char* default_val); // Parses a string for a bool flag, in the form of either @@ -64,9 +93,11 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value); bool IsFlag(const char* str, const char* flag); // Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or -// some non-alphanumeric character. As a special case, also returns true if -// value is the empty string. +// some non-alphanumeric character. Also returns false if the value matches +// one of 'no', 'false', 'off' (case-insensitive). As a special case, also +// returns true if value is the empty string. bool IsTruthyFlagValue(const std::string& value); + } // end namespace benchmark #endif // BENCHMARK_COMMANDLINEFLAGS_H_ diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc index 5460778c48..36bdb44595 100644 --- a/test/commandlineflags_gtest.cc +++ b/test/commandlineflags_gtest.cc @@ -34,6 +34,58 @@ TEST(BoolFromEnv, False) { ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "N", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "n", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "NO", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "No", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "no", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "F", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "f", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "FALSE", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "False", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "false", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "OFF", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Off", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "off", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", true), false); + unsetenv("BENCHMARK_IN_ENV"); } TEST(BoolFromEnv, True) { @@ -41,9 +93,63 @@ TEST(BoolFromEnv, True) { EXPECT_EQ(BoolFromEnv("in_env", false), true); unsetenv("BENCHMARK_IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Y", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "y", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "YES", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Yes", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "yes", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "T", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "t", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "TRUE", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "True", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "true", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "ON", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "On", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "on", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); + +#ifndef BENCHMARK_OS_WINDOWS + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "", 1), 0); + EXPECT_EQ(BoolFromEnv("in_env", false), true); + unsetenv("BENCHMARK_IN_ENV"); +#endif } TEST(Int32FromEnv, NotInEnv) { @@ -54,7 +160,7 @@ TEST(Int32FromEnv, NotInEnv) { TEST(Int32FromEnv, InvalidInteger) { ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); EXPECT_EQ(Int32FromEnv("in_env", 42), 42); - ASSERT_EQ(unsetenv("BENCHMARK_IN_ENV"), 0); + unsetenv("BENCHMARK_IN_ENV"); } TEST(Int32FromEnv, ValidInteger) { @@ -63,6 +169,23 @@ TEST(Int32FromEnv, ValidInteger) { unsetenv("BENCHMARK_IN_ENV"); } +TEST(DoubleFromEnv, NotInEnv) { + ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + EXPECT_EQ(DoubleFromEnv("not_in_env", 0.51), 0.51); +} + +TEST(DoubleFromEnv, InvalidReal) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + EXPECT_EQ(DoubleFromEnv("in_env", 0.51), 0.51); + unsetenv("BENCHMARK_IN_ENV"); +} + +TEST(DoubleFromEnv, ValidReal) { + ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0.51", 1), 0); + EXPECT_EQ(DoubleFromEnv("in_env", 0.71), 0.51); + unsetenv("BENCHMARK_IN_ENV"); +} + TEST(StringFromEnv, Default) { ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); EXPECT_STREQ(StringFromEnv("not_in_env", "foo"), "foo"); From d16ae64e5aeeda305a3f6d761eaf6a87dfd726bd Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 23 Oct 2019 04:13:40 -0400 Subject: [PATCH 182/330] Set CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS (#889) When building on Windows with `BUILD_SHARED_LIBS=ON`, the symbols were not being properly exported in the DLL. Fix this by setting `CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS`. Fixes #888 --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9db1361212..8cfe12552c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree buildi # in cases where it is not possible to build or find a valid version of gtest. option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF) function(should_enable_assembly_tests) if(CMAKE_BUILD_TYPE) From cf446a18bf37d5cc9b116f59cf9e6a9be89e58cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20Leitereg?= Date: Thu, 24 Oct 2019 21:13:03 +0200 Subject: [PATCH 183/330] Remove superfluous cache line scaling in JSON reporter. (#896) Cache size is already stored in bytes. --- src/json_reporter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 11db2b99d5..fe7b1be0d8 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -135,7 +135,7 @@ bool JSONReporter::ReportContext(const Context& context) { out << cache_indent << FormatKV("level", static_cast(CI.level)) << ",\n"; out << cache_indent - << FormatKV("size", static_cast(CI.size) * 1000u) << ",\n"; + << FormatKV("size", static_cast(CI.size)) << ",\n"; out << cache_indent << FormatKV("num_sharing", static_cast(CI.num_sharing)) << "\n"; From c50ac68c50ff8da3827cd6720792117910d85666 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Tue, 5 Nov 2019 20:46:13 +0100 Subject: [PATCH 184/330] CMake: use full add_test(NAME <> COMMAND <>) signature (#901) * CTest must use proper paths to executables With the following syntax: ``` add_test(NAME COMMAND [...]) ``` if `` specifies an executable target it will automatically be replaced by the location of the executable created at build time. This is important if a `_POSTFIX` like `_d` is used. * Fix typo in ctest invocation Instead of `-c` the uppercase `-C` must be used to select a config. But better use the longopt. --- appveyor.yml | 2 +- test/CMakeLists.txt | 50 ++++++++++++++++++++++----------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cf240190be..81da955f02 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -41,7 +41,7 @@ build_script: - cmake --build . --config %configuration% test_script: - - ctest -c %configuration% --timeout 300 --output-on-failure + - ctest --build-config %configuration% --timeout 300 --output-on-failure artifacts: - path: '_build/CMakeFiles/*.log' diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 030f35aae3..ddcb1a1eb9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,12 +54,12 @@ endmacro(compile_output_test) # Demonstration executable compile_benchmark_test(benchmark_test) -add_test(benchmark benchmark_test --benchmark_min_time=0.01) +add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) - add_test(${name}_list_only filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) add_filter_test(filter_simple "Foo" 3) @@ -82,16 +82,16 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(options_benchmarks options_test --benchmark_min_time=0.01) +add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) -add_test(basic_benchmark basic_test --benchmark_min_time=0.01) +add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) compile_benchmark_test(diagnostics_test) -add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) +add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) compile_benchmark_test(skip_with_error_test) -add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01) +add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) compile_benchmark_test(donotoptimize_test) # Some of the issues with DoNotOptimize only occur when optimization is enabled @@ -99,49 +99,49 @@ check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) +add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) compile_benchmark_test(fixture_test) -add_test(fixture_test fixture_test --benchmark_min_time=0.01) +add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) compile_benchmark_test(register_benchmark_test) -add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01) +add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) -add_test(map_test map_test --benchmark_min_time=0.01) +add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) compile_benchmark_test(multiple_ranges_test) -add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01) +add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) compile_benchmark_test_with_main(link_main_test) -add_test(link_main_test link_main_test --benchmark_min_time=0.01) +add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) compile_output_test(reporter_output_test) -add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) +add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) compile_output_test(templated_fixture_test) -add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01) +add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) compile_output_test(user_counters_test) -add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) +add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) compile_output_test(internal_threading_test) -add_test(internal_threading_test internal_threading_test --benchmark_min_time=0.01) +add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) compile_output_test(report_aggregates_only_test) -add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01) +add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(display_aggregates_only_test) -add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01) +add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(user_counters_tabular_test) -add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) compile_output_test(user_counters_thousands_test) -add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01) +add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) compile_output_test(memory_manager_test) -add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01) +add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) @@ -159,7 +159,7 @@ if (BENCHMARK_HAS_CXX03_FLAG) PROPERTIES LINK_FLAGS "-Wno-odr") endif() - add_test(cxx03 cxx03_test --benchmark_min_time=0.01) + add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) endif() # Attempt to work around flaky test failures when running on Appveyor servers. @@ -169,7 +169,7 @@ else() set(COMPLEXITY_MIN_TIME "0.01") endif() compile_output_test(complexity_test) -add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) ############################################################################### # GoogleTest Unit Tests @@ -184,7 +184,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(add_gtest name) compile_gtest(${name}) - add_test(${name} ${name}) + add_test(NAME ${name} COMMAND ${name}) endmacro() add_gtest(benchmark_gtest) From 173aff82ee1c93e4560f703157dcbd4f99887d3a Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Fri, 22 Nov 2019 15:06:43 +0300 Subject: [PATCH 185/330] src/counter.h: add header guard --- src/counter.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/counter.h b/src/counter.h index 1ad46d4940..b9ff2c4599 100644 --- a/src/counter.h +++ b/src/counter.h @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef COUNTER_H_ +#define COUNTER_H_ + #include "benchmark/benchmark.h" namespace benchmark { @@ -25,3 +28,5 @@ bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace internal } // end namespace benchmark + +#endif // COUNTER_H_ From 74e112ae9d2994e64d10ec68c7163fcec2e61a6d Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Fri, 22 Nov 2019 23:39:56 +0300 Subject: [PATCH 186/330] mingw.py: check for None via 'is', not '=='. --- mingw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mingw.py b/mingw.py index 706ad559db..65cf4b8745 100644 --- a/mingw.py +++ b/mingw.py @@ -204,7 +204,7 @@ def root(location = None, arch = None, version = None, threading = None, exceptions = 'sjlj' else: exceptions = keys[0] - if revision == None: + if revision is None: revision = max(versions[version][arch][threading][exceptions].keys()) if not location: location = os.path.join(tempfile.gettempdir(), 'mingw-builds') From cc7f50e126b13636f6203c549c7405ddacb5107f Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 23 Nov 2019 00:18:28 +0300 Subject: [PATCH 187/330] BenchmarkRunner: use std::lround() to round double to int From clang-tidy bugprone-incorrect-roundings check: > casting (double + 0.5) to integer leads to incorrect rounding; consider using lround (#include ) instead --- src/benchmark_runner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 0bae6a545e..337fac1b2b 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -264,7 +264,7 @@ class BenchmarkRunner { // So what seems to be the sufficiently-large iteration count? Round up. const IterationCount max_next_iters = - 0.5 + std::max(multiplier * i.iters, i.iters + 1.0); + std::lround(std::max(multiplier * i.iters, i.iters + 1.0)); // But we do have *some* sanity limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); From c22c266eaffbcfa213285655619edb3411c7916b Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 23 Nov 2019 00:19:02 +0300 Subject: [PATCH 188/330] JSONReporter: RoundDouble(): use std::lround() to round double to int From clang-tidy bugprone-incorrect-roundings check: > casting (double + 0.5) to integer leads to incorrect rounding; consider using lround (#include ) instead --- src/json_reporter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index fe7b1be0d8..e5f3c35248 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -92,7 +92,7 @@ std::string FormatKV(std::string const& key, double value) { return ss.str(); } -int64_t RoundDouble(double v) { return static_cast(v + 0.5); } +int64_t RoundDouble(double v) { return std::lround(v); } } // end namespace From 51d991f1d7e6b28b8280d44606c28e121df23db0 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 23 Nov 2019 00:23:11 +0300 Subject: [PATCH 189/330] ParseCommandLineFlags(): do not dereference argc if it is null Higher up we dereference argc only if it is not null. But here we do no such check. --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 07942eba65..b751b9c31f 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -435,7 +435,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; BenchmarkReporter::Context::executable_name = (argc && *argc > 0) ? argv[0] : "unknown"; - for (int i = 1; i < *argc; ++i) { + for (int i = 1; argc && i < *argc; ++i) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || From 49aa79b635239edc7dc5e3041d41405f850784a8 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 25 Nov 2019 13:05:13 +0000 Subject: [PATCH 190/330] update header guard to match style --- src/counter.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/counter.h b/src/counter.h index b9ff2c4599..1f5a58e31f 100644 --- a/src/counter.h +++ b/src/counter.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef COUNTER_H_ -#define COUNTER_H_ +#ifndef BENCHMARK_COUNTER_H_ +#define BENCHMARK_COUNTER_H_ #include "benchmark/benchmark.h" @@ -29,4 +29,4 @@ bool SameNames(UserCounters const& l, UserCounters const& r); } // end namespace benchmark -#endif // COUNTER_H_ +#endif // BENCHMARK_COUNTER_H_ From daf276ff940a0d6642c0c099b0e4a6ad7732eda8 Mon Sep 17 00:00:00 2001 From: Martin Blanchard Date: Sun, 1 Dec 2019 11:40:10 +0100 Subject: [PATCH 191/330] Document environment variables options usage (#894) See https://github.com/google/benchmark/issues/881 --- README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a8aa276f6c..62d3f4eacb 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Benchmark + [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) [![slackin](https://slackin-iqtfqnpzxd.now.sh/badge.svg)](https://slackin-iqtfqnpzxd.now.sh/) - A library to benchmark code snippets, similar to unit tests. Example: ```c++ @@ -149,7 +149,9 @@ this branch. However, this branch provides no stability guarantees and reserves the right to change and break the API at any time. ## Usage + ### Basic usage + Define a function that executes the code to measure, register it as a benchmark function using the `BENCHMARK` macro, and ensure an appropriate `main` function is available: @@ -239,15 +241,19 @@ too (`-lkstat`). ## User Guide ### Command Line + [Output Formats](#output-formats) [Output Files](#output-files) +[Running Benchmarks](#running-benchmarks) + [Running a Subset of Benchmarks](#running-a-subset-of-benchmarks) [Result Comparison](#result-comparison) ### Library + [Runtime and Reporting Considerations](#runtime-and-reporting-considerations) [Passing Arguments](#passing-arguments) @@ -282,17 +288,20 @@ too (`-lkstat`). [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) + ### Output Formats The library supports multiple output formats. Use the -`--benchmark_format=` flag to set the format type. `console` -is the default format. +`--benchmark_format=` (or set the +`BENCHMARK_FORMAT=` environment variable) flag to set +the format type. `console` is the default format. The Console format is intended to be a human readable format. By default the format generates color output. Context is output on stderr and the tabular data on stdout. Example tabular output looks like: + ``` Benchmark Time(ns) CPU(ns) Iterations ---------------------------------------------------------------------- @@ -306,6 +315,7 @@ The `context` attribute contains information about the run in general, including information about the CPU and the date. The `benchmarks` attribute contains a list of every benchmark run. Example json output looks like: + ```json { "context": { @@ -346,6 +356,7 @@ output looks like: The CSV format outputs comma-separated values. The `context` is output on stderr and the CSV itself on stdout. Example CSV output looks like: + ``` name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label "BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, @@ -357,16 +368,31 @@ name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label ### Output Files -Write benchmark results to a file with the `--benchmark_out=` option. -Specify the output format with `--benchmark_out_format={json|console|csv}`. Note that Specifying +Write benchmark results to a file with the `--benchmark_out=` option +(or set `BENCHMARK_OUT`). Specify the output format with +`--benchmark_out_format={json|console|csv}` (or set +`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that specifying `--benchmark_out` does not suppress the console output. + + +### Running Benchmarks + +Benchmarks are executed by running the produced binaries. Benchmarks binaries, +by default, accept options that may be specified either through their command +line interface or by setting environment variables before execution. For every +`--option_flag=` CLI swich, a corresponding environment variable +`OPTION_FLAG=` exist and is used as default if set (CLI switches always + prevails). A complete list of CLI options is available running benchmarks + with the `--help` switch. + ### Running a Subset of Benchmarks -The `--benchmark_filter=` option can be used to only run the benchmarks -which match the specified ``. For example: +The `--benchmark_filter=` option (or `BENCHMARK_FILTER=` +environment variable) can be used to only run the benchmarks that match +the specified ``. For example: ```bash $ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 @@ -384,7 +410,8 @@ BM_memcpy/32k 1834 ns 1837 ns 357143 ### Result comparison -It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) +It is possible to compare the benchmarking results. +See [Additional Tooling Documentation](docs/tools.md) @@ -450,6 +477,7 @@ range multiplier is changed to multiples of two. ```c++ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` + Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. The preceding code shows a method of defining a sparse range. The following @@ -466,6 +494,7 @@ static void BM_DenseRange(benchmark::State& state) { } BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128); ``` + Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ]. You might have a benchmark that depends on two or more inputs. For example, the @@ -535,6 +564,7 @@ void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { // the specified values to `extra_args`. BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); ``` + Note that elements of `...args` may refer to global variables. Users should avoid modifying global state inside of a benchmark. @@ -659,6 +689,7 @@ Also you can create templated fixture by using the following macros: * `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)` For example: + ```c++ template class MyFixture : public benchmark::Fixture {}; @@ -813,6 +844,7 @@ BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 ``` + Note above the additional header printed when the benchmark changes from ``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does not have the same counter set as ``BM_UserCounter``. @@ -1222,11 +1254,15 @@ the benchmark loop should be preferred. ### Disabling CPU Frequency Scaling + If you see this error: + ``` ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. ``` + you might want to disable the CPU frequency scaling while running the benchmark: + ```bash sudo cpupower frequency-set --governor performance ./mybench From 318d07113de67343127d0f980da7edfd763ab4a3 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sun, 1 Dec 2019 13:42:52 +0300 Subject: [PATCH 192/330] README.md: a few adjustments after #894 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 62d3f4eacb..d972ab050a 100644 --- a/README.md +++ b/README.md @@ -294,8 +294,8 @@ too (`-lkstat`). ### Output Formats The library supports multiple output formats. Use the -`--benchmark_format=` (or set the -`BENCHMARK_FORMAT=` environment variable) flag to set +`--benchmark_format=` flag (or set the +`BENCHMARK_FORMAT=` environment variable) to set the format type. `console` is the default format. The Console format is intended to be a human readable format. By default @@ -381,7 +381,7 @@ Write benchmark results to a file with the `--benchmark_out=` option Benchmarks are executed by running the produced binaries. Benchmarks binaries, by default, accept options that may be specified either through their command line interface or by setting environment variables before execution. For every -`--option_flag=` CLI swich, a corresponding environment variable +`--option_flag=` CLI switch, a corresponding environment variable `OPTION_FLAG=` exist and is used as default if set (CLI switches always prevails). A complete list of CLI options is available running benchmarks with the `--help` switch. From 367119482ff4abc3d73e4a109b410090fc281337 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 2 Dec 2019 12:29:16 +0300 Subject: [PATCH 193/330] CPU caches are binary units, not SI. (#911) As disscussed in https://github.com/google/benchmark/issues/899, it is all but certain that the multiplier should be 1024, not 1000. Fixes https://github.com/google/benchmark/issues/899 --- src/reporter.cc | 2 +- src/sysinfo.cc | 2 +- test/reporter_output_test.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/reporter.cc b/src/reporter.cc index 4d3e477d44..0b54fa421a 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -49,7 +49,7 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "CPU Caches:\n"; for (auto &CInfo : info.caches) { Out << " L" << CInfo.level << " " << CInfo.type << " " - << (CInfo.size / 1000) << "K"; + << (CInfo.size / 1024) << " KiB"; if (CInfo.num_sharing != 0) Out << " (x" << (info.num_cpus / CInfo.num_sharing) << ")"; Out << "\n"; diff --git a/src/sysinfo.cc b/src/sysinfo.cc index b5f99c7de5..5b7c4af780 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -270,7 +270,7 @@ std::vector GetCacheSizesFromKVFS() { else if (f && suffix != "K") PrintErrorAndDie("Invalid cache size format: Expected bytes ", suffix); else if (suffix == "K") - info.size *= 1000; + info.size *= 1024; } if (!ReadFromFile(StrCat(FPath, "type"), &info.type)) PrintErrorAndDie("Failed to read from file ", FPath, "type"); diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 8486d590dc..1a96b5f0bf 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -38,9 +38,9 @@ static int AddContextCases() { for (size_t I = 0; I < Caches.size(); ++I) { std::string num_caches_str = Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$"; - AddCases( - TC_ConsoleErr, - {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}}); + AddCases(TC_ConsoleErr, + {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str, + MR_Next}}); AddCases(TC_JSONOut, {{"\\{$", MR_Next}, {"\"type\": \"", MR_Next}, {"\"level\": %int,$", MR_Next}, From 0811f1d782455b3c80285bebf934a7045d845ed3 Mon Sep 17 00:00:00 2001 From: Tetsuo Kiso Date: Sun, 15 Dec 2019 19:38:54 +0900 Subject: [PATCH 194/330] Fix typo in mutex.h (#917) --- src/mutex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mutex.h b/src/mutex.h index 5f461d05a0..3fac79aea4 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -71,7 +71,7 @@ typedef std::condition_variable Condition; // NOTE: Wrappers for std::mutex and std::unique_lock are provided so that // we can annotate them with thread safety attributes and use the // -Wthread-safety warning with clang. The standard library types cannot be -// used directly because they do not provided the required annotations. +// used directly because they do not provide the required annotations. class CAPABILITY("mutex") Mutex { public: Mutex() {} From 5ce2429af7a8481581896afaa480552cc7584808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szit=C3=A1r=20Gerg=C5=91?= Date: Sun, 5 Jan 2020 13:32:40 +0100 Subject: [PATCH 195/330] Add d postfix to Debug libraries (#923) * Add DEBUG_POSTFIX to libraries. Makes it possible to install Debug and Release versions on the same system. Without this, there were only linker errors when using the wrong configuration. * Update CONTRIBUTORS and AUTHORS according to guide --- AUTHORS | 1 + CONTRIBUTORS | 1 + src/CMakeLists.txt | 2 ++ 3 files changed, 4 insertions(+) diff --git a/AUTHORS b/AUTHORS index 35c4c8cee9..06295c1ea0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -26,6 +26,7 @@ Eugene Zhuk Evgeny Safronov Federico Ficarelli Felix Homann +Gergő Szitár Google Inc. International Business Machines Corporation Ismael Jimenez Martinez diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 6b64a00628..04e8aa6a47 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -43,6 +43,7 @@ Evgeny Safronov Federico Ficarelli Felix Homann Geoffrey Martin-Noble +Gergő Szitár Hannes Hauswedell Ismael Jimenez Martinez Jern-Kuan Leong diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index eab1428e87..2650e3d196 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,6 +22,7 @@ set_target_properties(benchmark PROPERTIES OUTPUT_NAME "benchmark" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} + DEBUG_POSTFIX "d" ) target_include_directories(benchmark PUBLIC $ @@ -58,6 +59,7 @@ set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} + DEBUG_POSTFIX "d" ) target_include_directories(benchmark PUBLIC $ From daff5fead3fbe22c6fc58310ca3f49caf117f185 Mon Sep 17 00:00:00 2001 From: Jordan Williams Date: Tue, 14 Jan 2020 14:21:24 -0600 Subject: [PATCH 196/330] Alias CMake Targets. Fixes #921 (#926) * add Jordan Williams to both CONTRIBUTORS and AUTHORS * alias benchmark libraries Provide aliased CMake targets for the benchmark and benchmark_main targets. The alias targets are namespaced under benchmark::, which is the namespace when they are exported. I chose not to use either the PROJECT_NAME or the namespace variable but to hard-code the namespace. This is because the benchmark and benchmark_main targets are hard-coded by name themselves. Hard-coding the namespace is also much cleaner and easier to read. * link to aliased benchmark targets It is safer to link against namespaced targets because of how CMake interprets the double colon. Typo's will be caught by CMake at configuration-time instead of during compile / link time. * document the provided alias targets * add "Usage with CMake" section in documentation This section covers linking against the alias/import CMake targets and including them using either find_package or add_subdirectory. * format the "Usage with CMake" README section Added a newline after the "Usage with CMake" section header. Dropped the header level of the section by one to make it a direct subsection of the "Usage" section. Wrapped lines to be no longer than 80 characters in length. --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 24 ++++++++++++++++++++++-- src/CMakeLists.txt | 4 +++- test/CMakeLists.txt | 8 ++++---- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/AUTHORS b/AUTHORS index 06295c1ea0..89205a1adb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -33,6 +33,7 @@ Ismael Jimenez Martinez Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes +Jordan Williams Jussi Knuuttila Kaito Udagawa Kishan Kumar diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 04e8aa6a47..88f7eee06c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -50,6 +50,7 @@ Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes John Millikin +Jordan Williams Jussi Knuuttila Kai Wolf Kaito Udagawa diff --git a/README.md b/README.md index d972ab050a..560464eb88 100644 --- a/README.md +++ b/README.md @@ -178,8 +178,8 @@ BENCHMARK_MAIN(); ``` To run the benchmark, compile and link against the `benchmark` library -(libbenchmark.a/.so). If you followed the build steps above, this -library will be under the build directory you created. +(libbenchmark.a/.so). If you followed the build steps above, this library will +be under the build directory you created. ```bash # Example on linux after running the build steps above. Assumes the @@ -194,6 +194,26 @@ Alternatively, link against the `benchmark_main` library and remove The compiled executable will run all benchmarks by default. Pass the `--help` flag for option information or see the guide below. +### Usage with CMake + +If using CMake, it is recommended to link against the project-provided +`benchmark::benchmark` and `benchmark::benchmark_main` targets using +`target_link_libraries`. +It is possible to use ```find_package``` to import an installed version of the +library. +```cmake +find_package(benchmark REQUIRED) +``` +Alternatively, ```add_subdirectory``` will incorporate the library directly in +to one's CMake project. +```cmake +add_subdirectory(benchmark) +``` +Either way, link to the library as follows. +```cmake +target_link_libraries(MyTarget benchmark::benchmark) +``` + ## Platform Specific Build Instructions ### Building with GCC diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2650e3d196..81c902c5cd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ foreach(item ${BENCHMARK_MAIN}) endforeach() add_library(benchmark ${SOURCE_FILES}) +add_library(benchmark::benchmark ALIAS benchmark) set_target_properties(benchmark PROPERTIES OUTPUT_NAME "benchmark" VERSION ${GENERIC_LIB_VERSION} @@ -55,6 +56,7 @@ endif() # Benchmark main library add_library(benchmark_main "benchmark_main.cc") +add_library(benchmark::benchmark_main ALIAS benchmark_main) set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} @@ -64,7 +66,7 @@ set_target_properties(benchmark_main PROPERTIES target_include_directories(benchmark PUBLIC $ ) -target_link_libraries(benchmark_main benchmark) +target_link_libraries(benchmark_main benchmark::benchmark) set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ddcb1a1eb9..6afe47c400 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,17 +38,17 @@ add_library(output_test_helper STATIC output_test_helper.cc output_test.h) macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") - target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_benchmark_test) macro(compile_benchmark_test_with_main name) add_executable(${name} "${name}.cc") - target_link_libraries(${name} benchmark_main) + target_link_libraries(${name} benchmark::benchmark_main) endmacro(compile_benchmark_test_with_main) macro(compile_output_test name) add_executable(${name} "${name}.cc" output_test.h) - target_link_libraries(${name} output_test_helper benchmark + target_link_libraries(${name} output_test_helper benchmark::benchmark ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) @@ -178,7 +178,7 @@ add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time= if (BENCHMARK_ENABLE_GTEST_TESTS) macro(compile_gtest name) add_executable(${name} "${name}.cc") - target_link_libraries(${name} benchmark + target_link_libraries(${name} benchmark::benchmark gmock_main ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_gtest) From 5ac80de0379ae1153ca6ef141df89ecf53bf1110 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 30 Jan 2020 13:29:17 +0000 Subject: [PATCH 197/330] Disable pedantic warnings and errors until googletest/gmock is fixed --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cfe12552c..72466f9e81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,8 +143,9 @@ else() add_cxx_compiler_flag(-Werror RELEASE) add_cxx_compiler_flag(-Werror RELWITHDEBINFO) add_cxx_compiler_flag(-Werror MINSIZEREL) - add_cxx_compiler_flag(-pedantic) - add_cxx_compiler_flag(-pedantic-errors) + # Disabled until googletest (gmock) stops emitting variadic macro warnings + #add_cxx_compiler_flag(-pedantic) + #add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-Wshorten-64-to-32) add_cxx_compiler_flag(-fstrict-aliasing) # Disable warnings regarding deprecated parts of the library while building From e5ea03ce07997620980636b3ee7e6117d68bc6ad Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Fri, 31 Jan 2020 02:16:25 -0800 Subject: [PATCH 198/330] Fix cxx03 standard selection, option override in CMake 3.13+. Fixes #933 (#934) --- CMakeLists.txt | 1 + test/CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72466f9e81..67c0b70015 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ foreach(p CMP0056 # export EXE_LINKER_FLAGS to try_run CMP0057 # Support no if() IN_LIST operator CMP0063 # Honor visibility properties for all targets + CMP0077 # Allow option() overrides in importing projects ) if(POLICY ${p}) cmake_policy(SET ${p} NEW) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6afe47c400..0d228b85cb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -148,7 +148,8 @@ if (BENCHMARK_HAS_CXX03_FLAG) compile_benchmark_test(cxx03_test) set_target_properties(cxx03_test PROPERTIES - COMPILE_FLAGS "-std=c++03") + CXX_STANDARD 98 + CXX_STANDARD_REQUIRED YES) # libstdc++ provides different definitions within between dialects. When # LTO is enabled and -Werror is specified GCC diagnoses this ODR violation # causing the test to fail to compile. To prevent this we explicitly disable From 8982e1ee6aef31e48170400b7d1dc9969b156e5e Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Fri, 7 Feb 2020 23:48:01 +0000 Subject: [PATCH 199/330] Fix MSVC warning. (#935) This fixes the Visual Studio 2019 warning: `C4244: '=': conversion from 'int' to 'char', possible loss of data` When implicitly casting the return value of tolower() (int) to char. Fixes: #932 --- src/commandlineflags.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 4e60f0bd06..3380a127a8 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -217,8 +217,8 @@ bool IsTruthyFlagValue(const std::string& value) { !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); } else if (!value.empty()) { std::string value_lower(value); - std::transform(value_lower.begin(), value_lower.end(), - value_lower.begin(), ::tolower); + std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(), + [](char c) { return static_cast(::tolower(c)); }); return !(value_lower == "false" || value_lower == "no" || value_lower == "off"); } else From 168604d8f8e84c1f991de85f1a723c0df9a5b944 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 18 Feb 2020 07:04:45 -0500 Subject: [PATCH 200/330] [docs] Use `benchmark::IterationCount` rather than `int64_t` in lambda to complexity (#940) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 560464eb88..b43d4d20b9 100644 --- a/README.md +++ b/README.md @@ -622,7 +622,7 @@ that might be used to customize high-order term calculation. ```c++ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) - ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; }); + ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; }); ``` From c078337494086f9372a46b4ed31a3ae7b3f1a6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 21 Feb 2020 15:53:25 +0100 Subject: [PATCH 201/330] Relax CHECK condition in benchmark_runner.cc (#938) * Add State::error_occurred() * Relax CHECK condition in benchmark_runner.cc If the benchmark state contains an error, do not expect any iterations has been run. This allows using SkipWithError() and return early from the benchmark function. * README.md: document new possible usage of SkipWithError() --- README.md | 34 ++++++++++++++++++++++------------ include/benchmark/benchmark.h | 3 +++ src/benchmark_runner.cc | 2 +- test/skip_with_error_test.cc | 6 ++++++ 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b43d4d20b9..02a3bfad80 100644 --- a/README.md +++ b/README.md @@ -1184,7 +1184,9 @@ Users must explicitly exit the loop, otherwise all iterations will be performed. Users may explicitly return to exit the benchmark immediately. The `SkipWithError(...)` function may be used at any point within the benchmark, -including before and after the benchmark loop. +including before and after the benchmark loop. Moreover, if `SkipWithError(...)` +has been used, it is not required to reach the benchmark loop and one may return +from the benchmark function early. For example: @@ -1192,24 +1194,32 @@ For example: static void BM_test(benchmark::State& state) { auto resource = GetResource(); if (!resource.good()) { - state.SkipWithError("Resource is not good!"); - // KeepRunning() loop will not be entered. + state.SkipWithError("Resource is not good!"); + // KeepRunning() loop will not be entered. } while (state.KeepRunning()) { - auto data = resource.read_data(); - if (!resource.good()) { - state.SkipWithError("Failed to read data!"); - break; // Needed to skip the rest of the iteration. - } - do_stuff(data); + auto data = resource.read_data(); + if (!resource.good()) { + state.SkipWithError("Failed to read data!"); + break; // Needed to skip the rest of the iteration. + } + do_stuff(data); } } static void BM_test_ranged_fo(benchmark::State & state) { - state.SkipWithError("test will not be entered"); + auto resource = GetResource(); + if (!resource.good()) { + state.SkipWithError("Resource is not good!"); + return; // Early return is allowed when SkipWithError() has been used. + } for (auto _ : state) { - state.SkipWithError("Failed!"); - break; // REQUIRED to prevent all further iterations. + auto data = resource.read_data(); + if (!resource.good()) { + state.SkipWithError("Failed to read data!"); + break; // REQUIRED to prevent all further iterations. + } + do_stuff(data); } } ``` diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 144e212e1b..e5f6778958 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -541,6 +541,9 @@ class State { // responsibility to exit the scope as needed. void SkipWithError(const char* msg); + // Returns true if an error has been reported with 'SkipWithError(...)'. + bool error_occurred() const { return error_occurred_; } + // REQUIRES: called exactly once per iteration of the benchmarking loop. // Set the manually measured time for this benchmark iteration, which // is used instead of automatically measured time if UseManualTime() was diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 337fac1b2b..c414eff9a9 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -117,7 +117,7 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); State st = b->Run(iters, thread_id, &timer, manager); - CHECK(st.iterations() >= st.max_iterations) + CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { MutexLock l(manager->GetBenchmarkMutex()); diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 06579772ff..97a2e3c03b 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -61,6 +61,12 @@ int AddCases(const char* base_name, std::initializer_list const& v) { } // end namespace +void BM_error_no_running(benchmark::State& state) { + state.SkipWithError("error message"); +} +BENCHMARK(BM_error_no_running); +ADD_CASES("BM_error_no_running", {{"", true, "error message"}}); + void BM_error_before_running(benchmark::State& state) { state.SkipWithError("error message"); while (state.KeepRunning()) { From 8e0b1913d4ea803dfeb2e55567208fcab6b1b6c7 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 25 Feb 2020 11:21:58 +0000 Subject: [PATCH 202/330] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 32 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..6c2ced9b2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**System** +Which OS, compiler, and compiler version are you using: + - OS: + - Compiler and version: + +**To reproduce** +Steps to reproduce the behavior: +1. sync to commit ... +2. cmake/bazel... +3. make ... +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..9e8ab6a673 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FR]" +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 70d89ac5190923bdaebf9d0a4ac04085796d062a Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Mon, 16 Mar 2020 14:22:16 +0300 Subject: [PATCH 203/330] Revert "Add d postfix to Debug libraries (#923)" This reverts commit 5ce2429af7a8481581896afaa480552cc7584808. Reverts https://github.com/google/benchmark/pull/923 Reopens https://github.com/google/benchmark/issues/922 Fixes https://github.com/google/benchmark/issues/928 Closes https://github.com/google/benchmark/pull/930 See discussion in https://github.com/google/benchmark/issues/928 this broke pkg-config support, since there we don't account for the suffix, nor is it trivial to do so. --- src/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81c902c5cd..35d559eeae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,7 +23,6 @@ set_target_properties(benchmark PROPERTIES OUTPUT_NAME "benchmark" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} - DEBUG_POSTFIX "d" ) target_include_directories(benchmark PUBLIC $ @@ -61,7 +60,6 @@ set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} - DEBUG_POSTFIX "d" ) target_include_directories(benchmark PUBLIC $ From b23d35573bb9f33f9b76cadc652bb7ef88c6c64d Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Mon, 30 Mar 2020 09:22:37 +0300 Subject: [PATCH 204/330] Fix Travis-ci builds (#950) Line "- /usr/local/bin:$PATH" is misformatted. It must be something like "- PATH=/usr/local/bin:$PATH". It seems something changed in tarvis-ci month ago and now this leads to: Setting environment variables from .travis.yml $ export PATH= Defailt PATH is /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin so already containts /usr/local/bin. Image "xcode8.3" contains macOS 10.12 (sierra) which has no bottles with precompiled gcc-7 in homebrew storage. Image "xcode9.4" is a current default with macOS 10.13 (high_sierra). Link: https://docs.travis-ci.com/user/reference/osx/ Link: https://formulae.brew.sh/formula/gcc@7 Signed-off-by: Konstantin Khlebnikov --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b6cfc7046..f220a7df61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,6 @@ sudo: required dist: trusty language: cpp -env: - global: - - /usr/local/bin:$PATH - matrix: include: - compiler: gcc @@ -169,7 +165,7 @@ matrix: - BUILD_32_BITS=ON - EXTRA_FLAGS="-m32" - os: osx - osx_image: xcode8.3 + osx_image: xcode9.4 compiler: gcc env: - COMPILER=g++-7 C_COMPILER=gcc-7 BUILD_TYPE=Debug From 0ab2c2906b95f41dfa7b692ca4ab856e20ebd3d4 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 6 Apr 2020 13:52:09 +0100 Subject: [PATCH 205/330] Fix type conversion warnings. (#951) * Fix type conversion warnings. Fixes #949 Tested locally (Linux/clang), but warnings are on MSVC so may differ. * Drop the ULP so the double test passes --- src/benchmark_runner.cc | 5 +++-- test/statistics_gtest.cc | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index c414eff9a9..7bc6b6329e 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -263,8 +263,9 @@ class BenchmarkRunner { if (multiplier <= 1.0) multiplier = 2.0; // So what seems to be the sufficiently-large iteration count? Round up. - const IterationCount max_next_iters = - std::lround(std::max(multiplier * i.iters, i.iters + 1.0)); + const IterationCount max_next_iters = static_cast( + std::lround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); // But we do have *some* sanity limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); diff --git a/test/statistics_gtest.cc b/test/statistics_gtest.cc index 99e314920c..3ddc72dd7a 100644 --- a/test/statistics_gtest.cc +++ b/test/statistics_gtest.cc @@ -21,8 +21,8 @@ TEST(StatisticsTest, Median) { TEST(StatisticsTest, StdDev) { EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({1, 2, 3}), 1.0); - EXPECT_FLOAT_EQ(benchmark::StatisticsStdDev({1.5, 2.4, 3.3, 4.2, 5.1}), - 1.42302495); + EXPECT_DOUBLE_EQ(benchmark::StatisticsStdDev({2.5, 2.4, 3.3, 4.2, 5.1}), + 1.151086443322134); } } // end namespace From a77d5f70efaebe2b7e8c10134526a23a7ce7ef35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Marques?= Date: Fri, 10 Apr 2020 17:02:13 +0100 Subject: [PATCH 206/330] Fix cycleclock::Now for RISC-V and PPC (#955) Fixes the following issues with the implementation of `cycleclock::Now`: - The RISC-V implementation wouldn't compile due to a typo; - Both the PPC and RISC-V implementation's asm statements lacked the volatile keyword. This resulted in the repeated read of the counter's high part being optimized away, so overflow wasn't handled at all. Multiple counter reads could also be misoptimized, especially in LTO scenarios. - Relied on the zero/sign-extension of inline asm operands, which isn't guaranteed to occur and differs between compilers, namely GCC and Clang. The PowerPC64 implementation was improved to do a single 64-bit read of the time-base counter. The RISC-V implementation was improved to do the overflow handing in assembly, since Clang would generate a branch, defeating the purpose of the non-branching counter reading approach. --- src/cycleclock.h | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/cycleclock.h b/src/cycleclock.h index d5d62c4c7f..3901da6a86 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -84,13 +84,21 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { return (high << 32) | low; #elif defined(__powerpc__) || defined(__ppc__) // This returns a time-base, which is not always precisely a cycle-count. - int64_t tbl, tbu0, tbu1; - asm("mftbu %0" : "=r"(tbu0)); - asm("mftb %0" : "=r"(tbl)); - asm("mftbu %0" : "=r"(tbu1)); - tbl &= -static_cast(tbu0 == tbu1); - // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is garbage) - return (tbu1 << 32) | tbl; +#if defined(__powerpc64__) || defined(__ppc64__) + int64_t tb; + asm volatile("mfspr %0, 268" : "=r" (tb)); + return tb; +#else + uint32_t tbl, tbu0, tbu1; + asm volatile( + "mftbu %0\n" + "mftbl %1\n" + "mftbu %2" + : "=r"(tbu0), "=r"(tbl), "=r"(tbu1)); + tbl &= -static_cast(tbu0 == tbu1); + // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is no longer needed) + return (static_cast(tbu1) << 32) | tbl; +#endif #elif defined(__sparc__) int64_t tick; asm(".byte 0x83, 0x41, 0x00, 0x00"); @@ -167,16 +175,22 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #elif defined(__riscv) // RISC-V // Use RDCYCLE (and RDCYCLEH on riscv32) #if __riscv_xlen == 32 - uint64_t cycles_low, cycles_hi0, cycles_hi1; - asm("rdcycleh %0" : "=r"(cycles_hi0)); - asm("rdcycle %0" : "=r"(cycles_lo)); - asm("rdcycleh %0" : "=r"(cycles_hi1)); - // This matches the PowerPC overflow detection, above - cycles_lo &= -static_cast(cycles_hi0 == cycles_hi1); - return (cycles_hi1 << 32) | cycles_lo; + uint32_t cycles_lo, cycles_hi0, cycles_hi1; + // This asm also includes the PowerPC overflow handling strategy, as above. + // Implemented in assembly because Clang insisted on branching. + asm volatile( + "rdcycleh %0\n" + "rdcycle %1\n" + "rdcycleh %2\n" + "sub %0, %0, %2\n" + "seqz %0, %0\n" + "sub %0, zero, %0\n" + "and %1, %1, %0\n" + : "=r"(cycles_hi0), "=r"(cycles_lo), "=r"(cycles_hi1)); + return (static_cast(cycles_hi1) << 32) | cycles_lo; #else uint64_t cycles; - asm("rdcycle %0" : "=r"(cycles)); + asm volatile("rdcycle %0" : "=r"(cycles)); return cycles; #endif #else From 8cead007830bdbe94b7cc259e873179d0ef84da6 Mon Sep 17 00:00:00 2001 From: Keith Moyer Date: Tue, 14 Apr 2020 02:20:22 -0700 Subject: [PATCH 207/330] Remove warnings for internal use of CSVReporter (#956) In a previous commit[1], diagnostic pragmas were used to avoid this warning. However, the incorrect warning flag was indicated, leaving the warning in place. -Wdeprecated is for deprecated features while -Wdeprecated-declarations for deprecated functions, variables, and types[2]. [1] c408461983dd3adf49d450d7db926fc46f1d99a0 [2] https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html --- src/benchmark.cc | 4 ++-- test/output_test_helper.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index b751b9c31f..1c049f2884 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -284,10 +284,10 @@ void RunBenchmarks(const std::vector& benchmarks, } // Disable deprecated warnings temporarily because we need to reference -// CSVReporter but don't want to trigger -Werror=-Wdeprecated +// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations #ifdef __GNUC__ #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif std::unique_ptr CreateReporter( diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index bdb34c8796..f99b3a8261 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -374,10 +374,10 @@ int SetSubstitutions( } // Disable deprecated warnings temporarily because we need to reference -// CSVReporter but don't want to trigger -Werror=-Wdeprecated +// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations #ifdef __GNUC__ #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif void RunOutputTests(int argc, char* argv[]) { using internal::GetTestCaseList; From ecc1685340f58f7fe6b707036bc0bb1fccabb0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Marques?= Date: Fri, 17 Apr 2020 17:31:49 +0100 Subject: [PATCH 208/330] Fix formatting issues introduced by a77d5f7 (#959) --- src/cycleclock.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/cycleclock.h b/src/cycleclock.h index 3901da6a86..179c67cd61 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -86,15 +86,15 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // This returns a time-base, which is not always precisely a cycle-count. #if defined(__powerpc64__) || defined(__ppc64__) int64_t tb; - asm volatile("mfspr %0, 268" : "=r" (tb)); + asm volatile("mfspr %0, 268" : "=r"(tb)); return tb; #else uint32_t tbl, tbu0, tbu1; asm volatile( - "mftbu %0\n" - "mftbl %1\n" - "mftbu %2" - : "=r"(tbu0), "=r"(tbl), "=r"(tbu1)); + "mftbu %0\n" + "mftbl %1\n" + "mftbu %2" + : "=r"(tbu0), "=r"(tbl), "=r"(tbu1)); tbl &= -static_cast(tbu0 == tbu1); // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is no longer needed) return (static_cast(tbu1) << 32) | tbl; @@ -179,14 +179,14 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // This asm also includes the PowerPC overflow handling strategy, as above. // Implemented in assembly because Clang insisted on branching. asm volatile( - "rdcycleh %0\n" - "rdcycle %1\n" - "rdcycleh %2\n" - "sub %0, %0, %2\n" - "seqz %0, %0\n" - "sub %0, zero, %0\n" - "and %1, %1, %0\n" - : "=r"(cycles_hi0), "=r"(cycles_lo), "=r"(cycles_hi1)); + "rdcycleh %0\n" + "rdcycle %1\n" + "rdcycleh %2\n" + "sub %0, %0, %2\n" + "seqz %0, %0\n" + "sub %0, zero, %0\n" + "and %1, %1, %0\n" + : "=r"(cycles_hi0), "=r"(cycles_lo), "=r"(cycles_hi1)); return (static_cast(cycles_hi1) << 32) | cycles_lo; #else uint64_t cycles; From 56898e9a92fba537671d5462df9c5ef2ea6a823a Mon Sep 17 00:00:00 2001 From: Kai Wolf Date: Thu, 23 Apr 2020 13:19:19 +0200 Subject: [PATCH 209/330] Add missing header include - fixes Android build (#960) * Add missing header This commit fixes a current build error on Android where 'errno' is an unidentified symbol due to a missing header * Update string_util.cc Conditionally adds if BENCHMARK_STL_ANDROID_GNUSTL is defined --- src/string_util.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/string_util.cc b/src/string_util.cc index 39b01a1719..ac60b5588f 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -1,6 +1,9 @@ #include "string_util.h" #include +#ifdef BENCHMARK_STL_ANDROID_GNUSTL +#include +#endif #include #include #include From d3ad0b9d11c190cb58de5fb17c3555def61fdc96 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Wed, 6 May 2020 17:28:29 +0100 Subject: [PATCH 210/330] Add Python bindings. (#957) * Add Python bindings. * Add license headers. * Change example to a test. * Add example usage to module docstring. --- WORKSPACE | 15 +++ bindings/python/BUILD | 3 + bindings/python/benchmark/BUILD | 38 ++++++++ bindings/python/benchmark/__init__.py | 62 +++++++++++++ bindings/python/benchmark/benchmark.cc | 47 ++++++++++ bindings/python/benchmark/example.py | 32 +++++++ bindings/python/build_defs.bzl | 25 +++++ bindings/python/pybind11.BUILD | 20 ++++ bindings/python/python_headers.BUILD | 6 ++ bindings/python/requirements.txt | 2 + setup.py | 124 +++++++++++++++++++++++++ 11 files changed, 374 insertions(+) create mode 100644 bindings/python/BUILD create mode 100644 bindings/python/benchmark/BUILD create mode 100644 bindings/python/benchmark/__init__.py create mode 100644 bindings/python/benchmark/benchmark.cc create mode 100644 bindings/python/benchmark/example.py create mode 100644 bindings/python/build_defs.bzl create mode 100644 bindings/python/pybind11.BUILD create mode 100644 bindings/python/python_headers.BUILD create mode 100644 bindings/python/requirements.txt create mode 100644 setup.py diff --git a/WORKSPACE b/WORKSPACE index 8df248a4c0..dc6ea02b95 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,3 +13,18 @@ http_archive( strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], ) + +http_archive( + name = "pybind11", + build_file = "@//bindings/python:pybind11.BUILD", + sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", + strip_prefix = "pybind11-2.4.3", + urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], +) + +new_local_repository( + name = "python_headers", + build_file = "@//bindings/python:python_headers.BUILD", + path = "/usr/include/python3.6", # May be overwritten by setup.py. +) + diff --git a/bindings/python/BUILD b/bindings/python/BUILD new file mode 100644 index 0000000000..9559a76b30 --- /dev/null +++ b/bindings/python/BUILD @@ -0,0 +1,3 @@ +exports_files(glob(["*.BUILD"])) +exports_files(["build_defs.bzl"]) + diff --git a/bindings/python/benchmark/BUILD b/bindings/python/benchmark/BUILD new file mode 100644 index 0000000000..49f536e615 --- /dev/null +++ b/bindings/python/benchmark/BUILD @@ -0,0 +1,38 @@ +load("//bindings/python:build_defs.bzl", "py_extension") + +py_library( + name = "benchmark", + srcs = ["__init__.py"], + visibility = ["//visibility:public"], + deps = [ + ":_benchmark", + # pip; absl:app + ], +) + +py_extension( + name = "_benchmark", + srcs = ["benchmark.cc"], + copts = [ + "-fexceptions", + "-fno-strict-aliasing", + ], + features = ["-use_header_modules"], + deps = [ + "//:benchmark", + "@pybind11", + "@python_headers", + ], +) + +py_test( + name = "example", + srcs = ["example.py"], + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":benchmark", + ], +) + diff --git a/bindings/python/benchmark/__init__.py b/bindings/python/benchmark/__init__.py new file mode 100644 index 0000000000..27f76e0567 --- /dev/null +++ b/bindings/python/benchmark/__init__.py @@ -0,0 +1,62 @@ +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python benchmarking utilities. + +Example usage: + import benchmark + + @benchmark.register + def my_benchmark(state): + ... # Code executed outside `while` loop is not timed. + + while state: + ... # Code executed within `while` loop is timed. + + if __name__ == '__main__': + benchmark.main() +""" + +from absl import app +from benchmark import _benchmark + +__all__ = [ + "register", + "main", +] + +__version__ = "0.1.0" + + +def register(f=None, *, name=None): + if f is None: + return lambda f: register(f, name=name) + if name is None: + name = f.__name__ + _benchmark.RegisterBenchmark(name, f) + return f + + +def _flags_parser(argv): + argv = _benchmark.Initialize(argv) + return app.parse_flags_with_usage(argv) + + +def _run_benchmarks(argv): + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + return _benchmark.RunSpecifiedBenchmarks() + + +def main(argv=None): + return app.run(_run_benchmarks, argv=argv, flags_parser=_flags_parser) diff --git a/bindings/python/benchmark/benchmark.cc b/bindings/python/benchmark/benchmark.cc new file mode 100644 index 0000000000..ef95559659 --- /dev/null +++ b/bindings/python/benchmark/benchmark.cc @@ -0,0 +1,47 @@ +// Benchmark for Python. + +#include "benchmark/benchmark.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace { +namespace py = ::pybind11; + +std::vector Initialize(const std::vector& argv) { + // The `argv` pointers here become invalid when this function returns, but + // benchmark holds the pointer to `argv[0]`. We create a static copy of it + // so it persists, and replace the pointer below. + static std::string executable_name(argv[0]); + std::vector ptrs; + ptrs.reserve(argv.size()); + for (auto& arg : argv) { + ptrs.push_back(const_cast(arg.c_str())); + } + ptrs[0] = const_cast(executable_name.c_str()); + int argc = static_cast(argv.size()); + benchmark::Initialize(&argc, ptrs.data()); + std::vector remaining_argv; + remaining_argv.reserve(argc); + for (int i = 0; i < argc; ++i) { + remaining_argv.emplace_back(ptrs[i]); + } + return remaining_argv; +} + +void RegisterBenchmark(const char* name, py::function f) { + benchmark::RegisterBenchmark(name, [f](benchmark::State& state) { + f(&state); + }); +} + +PYBIND11_MODULE(_benchmark, m) { + m.def("Initialize", Initialize); + m.def("RegisterBenchmark", RegisterBenchmark); + m.def("RunSpecifiedBenchmarks", + []() { benchmark::RunSpecifiedBenchmarks(); }); + + py::class_(m, "State") + .def("__bool__", &benchmark::State::KeepRunning) + .def_property_readonly("keep_running", &benchmark::State::KeepRunning); +}; +} // namespace diff --git a/bindings/python/benchmark/example.py b/bindings/python/benchmark/example.py new file mode 100644 index 0000000000..24da12780a --- /dev/null +++ b/bindings/python/benchmark/example.py @@ -0,0 +1,32 @@ +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Example of Python using C++ benchmark framework.""" + +import benchmark + + +@benchmark.register +def empty(state): + while state: + pass + + +@benchmark.register +def sum_million(state): + while state: + sum(range(1_000_000)) + + +if __name__ == '__main__': + benchmark.main() diff --git a/bindings/python/build_defs.bzl b/bindings/python/build_defs.bzl new file mode 100644 index 0000000000..45907aaa5e --- /dev/null +++ b/bindings/python/build_defs.bzl @@ -0,0 +1,25 @@ +_SHARED_LIB_SUFFIX = { + "//conditions:default": ".so", + "//:windows": ".dll", +} + +def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []): + for shared_lib_suffix in _SHARED_LIB_SUFFIX.values(): + shared_lib_name = name + shared_lib_suffix + native.cc_binary( + name = shared_lib_name, + linkshared = 1, + linkstatic = 1, + srcs = srcs + hdrs, + copts = copts, + features = features, + deps = deps, + ) + + return native.py_library( + name = name, + data = select({ + platform: [name + shared_lib_suffix] + for platform, shared_lib_suffix in _SHARED_LIB_SUFFIX.items() + }), + ) diff --git a/bindings/python/pybind11.BUILD b/bindings/python/pybind11.BUILD new file mode 100644 index 0000000000..bc83350038 --- /dev/null +++ b/bindings/python/pybind11.BUILD @@ -0,0 +1,20 @@ +cc_library( + name = "pybind11", + hdrs = glob( + include = [ + "include/pybind11/*.h", + "include/pybind11/detail/*.h", + ], + exclude = [ + "include/pybind11/common.h", + "include/pybind11/eigen.h", + ], + ), + copts = [ + "-fexceptions", + "-Wno-undefined-inline", + "-Wno-pragma-once-outside-header", + ], + includes = ["include"], + visibility = ["//visibility:public"], +) diff --git a/bindings/python/python_headers.BUILD b/bindings/python/python_headers.BUILD new file mode 100644 index 0000000000..9c34cf6ca4 --- /dev/null +++ b/bindings/python/python_headers.BUILD @@ -0,0 +1,6 @@ +cc_library( + name = "python_headers", + hdrs = glob(["**/*.h"]), + includes = ["."], + visibility = ["//visibility:public"], +) diff --git a/bindings/python/requirements.txt b/bindings/python/requirements.txt new file mode 100644 index 0000000000..f5bbe7eca5 --- /dev/null +++ b/bindings/python/requirements.txt @@ -0,0 +1,2 @@ +absl-py>=0.7.1 + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..f4c063346a --- /dev/null +++ b/setup.py @@ -0,0 +1,124 @@ +import os +import posixpath +import re +import shutil +import sys + +from distutils import sysconfig +import setuptools +from setuptools.command import build_ext + + +here = os.path.dirname(os.path.abspath(__file__)) + + +IS_WINDOWS = sys.platform.startswith('win') + + +def _get_version(): + """Parse the version string from __init__.py.""" + with open(os.path.join(here, 'bindings', 'python', 'benchmark', '__init__.py')) as f: + try: + version_line = next( + line for line in f if line.startswith('__version__')) + except StopIteration: + raise ValueError('__version__ not defined in __init__.py') + else: + ns = {} + exec(version_line, ns) # pylint: disable=exec-used + return ns['__version__'] + + +def _parse_requirements(path): + with open(os.path.join(here, path)) as f: + return [ + line.rstrip() for line in f + if not (line.isspace() or line.startswith('#')) + ] + + +class BazelExtension(setuptools.Extension): + """A C/C++ extension that is defined as a Bazel BUILD target.""" + + def __init__(self, name, bazel_target): + self.bazel_target = bazel_target + self.relpath, self.target_name = ( + posixpath.relpath(bazel_target, '//').split(':')) + setuptools.Extension.__init__(self, name, sources=[]) + + +class BuildBazelExtension(build_ext.build_ext): + """A command that runs Bazel to build a C/C++ extension.""" + + def run(self): + for ext in self.extensions: + self.bazel_build(ext) + build_ext.build_ext.run(self) + + def bazel_build(self, ext): + with open('WORKSPACE', 'r') as f: + workspace_contents = f.read() + + with open('WORKSPACE', 'w') as f: + f.write(re.sub( + r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', + sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), + workspace_contents)) + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + bazel_argv = [ + 'bazel', + 'build', + ext.bazel_target, + '--symlink_prefix=' + os.path.join(self.build_temp, 'bazel-'), + '--compilation_mode=' + ('dbg' if self.debug else 'opt'), + ] + + if IS_WINDOWS: + # Link with python*.lib. + for library_dir in self.library_dirs: + bazel_argv.append('--linkopt=/LIBPATH:' + library_dir) + + self.spawn(bazel_argv) + + shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' + ext_bazel_bin_path = os.path.join( + self.build_temp, 'bazel-bin', + ext.relpath, ext.target_name + shared_lib_suffix) + ext_dest_path = self.get_ext_fullpath(ext.name) + ext_dest_dir = os.path.dirname(ext_dest_path) + if not os.path.exists(ext_dest_dir): + os.makedirs(ext_dest_dir) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + + +setuptools.setup( + name='google-benchmark', + version=_get_version(), + url='https://github.com/google/benchmark', + description='A library to benchmark code snippets.', + author='Google', + author_email='benchmark-py@google.com', + # Contained modules and scripts. + package_dir={'': 'bindings/python'}, + packages=setuptools.find_packages('bindings/python'), + install_requires=_parse_requirements('bindings/python/requirements.txt'), + cmdclass=dict(build_ext=BuildBazelExtension), + ext_modules=[BazelExtension('benchmark._benchmark', '//bindings/python/benchmark:_benchmark')], + zip_safe=False, + # PyPI package information. + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Software Development :: Testing', + 'Topic :: System :: Benchmark', + ], + license='Apache 2.0', + keywords='benchmark', +) From 9284e90f28a9815f0b0954005bb6bab4283a9248 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 26 May 2020 23:20:07 +0200 Subject: [PATCH 211/330] Drop unused mingw.py (#966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use ==/!= to compare constant literals (str, bytes, int, float, tuple) Avoid Syntax Warning on Python 3.8: `3 is 3` # —> Syntax Warning * Delete mingw.py --- mingw.py | 320 ------------------------------------------------------- 1 file changed, 320 deletions(-) delete mode 100644 mingw.py diff --git a/mingw.py b/mingw.py deleted file mode 100644 index 65cf4b8745..0000000000 --- a/mingw.py +++ /dev/null @@ -1,320 +0,0 @@ -#! /usr/bin/env python -# encoding: utf-8 - -import argparse -import errno -import logging -import os -import platform -import re -import sys -import subprocess -import tempfile - -try: - import winreg -except ImportError: - import _winreg as winreg -try: - import urllib.request as request -except ImportError: - import urllib as request -try: - import urllib.parse as parse -except ImportError: - import urlparse as parse - -class EmptyLogger(object): - ''' - Provides an implementation that performs no logging - ''' - def debug(self, *k, **kw): - pass - def info(self, *k, **kw): - pass - def warn(self, *k, **kw): - pass - def error(self, *k, **kw): - pass - def critical(self, *k, **kw): - pass - def setLevel(self, *k, **kw): - pass - -urls = ( - 'http://downloads.sourceforge.net/project/mingw-w64/Toolchains%20' - 'targetting%20Win32/Personal%20Builds/mingw-builds/installer/' - 'repository.txt', - 'http://downloads.sourceforge.net/project/mingwbuilds/host-windows/' - 'repository.txt' -) -''' -A list of mingw-build repositories -''' - -def repository(urls = urls, log = EmptyLogger()): - ''' - Downloads and parse mingw-build repository files and parses them - ''' - log.info('getting mingw-builds repository') - versions = {} - re_sourceforge = re.compile(r'http://sourceforge.net/projects/([^/]+)/files') - re_sub = r'http://downloads.sourceforge.net/project/\1' - for url in urls: - log.debug(' - requesting: %s', url) - socket = request.urlopen(url) - repo = socket.read() - if not isinstance(repo, str): - repo = repo.decode(); - socket.close() - for entry in repo.split('\n')[:-1]: - value = entry.split('|') - version = tuple([int(n) for n in value[0].strip().split('.')]) - version = versions.setdefault(version, {}) - arch = value[1].strip() - if arch == 'x32': - arch = 'i686' - elif arch == 'x64': - arch = 'x86_64' - arch = version.setdefault(arch, {}) - threading = arch.setdefault(value[2].strip(), {}) - exceptions = threading.setdefault(value[3].strip(), {}) - revision = exceptions.setdefault(int(value[4].strip()[3:]), - re_sourceforge.sub(re_sub, value[5].strip())) - return versions - -def find_in_path(file, path=None): - ''' - Attempts to find an executable in the path - ''' - if platform.system() == 'Windows': - file += '.exe' - if path is None: - path = os.environ.get('PATH', '') - if type(path) is type(''): - path = path.split(os.pathsep) - return list(filter(os.path.exists, - map(lambda dir, file=file: os.path.join(dir, file), path))) - -def find_7zip(log = EmptyLogger()): - ''' - Attempts to find 7zip for unpacking the mingw-build archives - ''' - log.info('finding 7zip') - path = find_in_path('7z') - if not path: - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\7-Zip') - path, _ = winreg.QueryValueEx(key, 'Path') - path = [os.path.join(path, '7z.exe')] - log.debug('found \'%s\'', path[0]) - return path[0] - -find_7zip() - -def unpack(archive, location, log = EmptyLogger()): - ''' - Unpacks a mingw-builds archive - ''' - sevenzip = find_7zip(log) - log.info('unpacking %s', os.path.basename(archive)) - cmd = [sevenzip, 'x', archive, '-o' + location, '-y'] - log.debug(' - %r', cmd) - with open(os.devnull, 'w') as devnull: - subprocess.check_call(cmd, stdout = devnull) - -def download(url, location, log = EmptyLogger()): - ''' - Downloads and unpacks a mingw-builds archive - ''' - log.info('downloading MinGW') - log.debug(' - url: %s', url) - log.debug(' - location: %s', location) - - re_content = re.compile(r'attachment;[ \t]*filename=(")?([^"]*)(")?[\r\n]*') - - stream = request.urlopen(url) - try: - content = stream.getheader('Content-Disposition') or '' - except AttributeError: - content = stream.headers.getheader('Content-Disposition') or '' - matches = re_content.match(content) - if matches: - filename = matches.group(2) - else: - parsed = parse.urlparse(stream.geturl()) - filename = os.path.basename(parsed.path) - - try: - os.makedirs(location) - except OSError as e: - if e.errno == errno.EEXIST and os.path.isdir(location): - pass - else: - raise - - archive = os.path.join(location, filename) - with open(archive, 'wb') as out: - while True: - buf = stream.read(1024) - if not buf: - break - out.write(buf) - unpack(archive, location, log = log) - os.remove(archive) - - possible = os.path.join(location, 'mingw64') - if not os.path.exists(possible): - possible = os.path.join(location, 'mingw32') - if not os.path.exists(possible): - raise ValueError('Failed to find unpacked MinGW: ' + possible) - return possible - -def root(location = None, arch = None, version = None, threading = None, - exceptions = None, revision = None, log = EmptyLogger()): - ''' - Returns the root folder of a specific version of the mingw-builds variant - of gcc. Will download the compiler if needed - ''' - - # Get the repository if we don't have all the information - if not (arch and version and threading and exceptions and revision): - versions = repository(log = log) - - # Determine some defaults - version = version or max(versions.keys()) - if not arch: - arch = platform.machine().lower() - if arch == 'x86': - arch = 'i686' - elif arch == 'amd64': - arch = 'x86_64' - if not threading: - keys = versions[version][arch].keys() - if 'posix' in keys: - threading = 'posix' - elif 'win32' in keys: - threading = 'win32' - else: - threading = keys[0] - if not exceptions: - keys = versions[version][arch][threading].keys() - if 'seh' in keys: - exceptions = 'seh' - elif 'sjlj' in keys: - exceptions = 'sjlj' - else: - exceptions = keys[0] - if revision is None: - revision = max(versions[version][arch][threading][exceptions].keys()) - if not location: - location = os.path.join(tempfile.gettempdir(), 'mingw-builds') - - # Get the download url - url = versions[version][arch][threading][exceptions][revision] - - # Tell the user whatzzup - log.info('finding MinGW %s', '.'.join(str(v) for v in version)) - log.debug(' - arch: %s', arch) - log.debug(' - threading: %s', threading) - log.debug(' - exceptions: %s', exceptions) - log.debug(' - revision: %s', revision) - log.debug(' - url: %s', url) - - # Store each specific revision differently - slug = '{version}-{arch}-{threading}-{exceptions}-rev{revision}' - slug = slug.format( - version = '.'.join(str(v) for v in version), - arch = arch, - threading = threading, - exceptions = exceptions, - revision = revision - ) - if arch == 'x86_64': - root_dir = os.path.join(location, slug, 'mingw64') - elif arch == 'i686': - root_dir = os.path.join(location, slug, 'mingw32') - else: - raise ValueError('Unknown MinGW arch: ' + arch) - - # Download if needed - if not os.path.exists(root_dir): - downloaded = download(url, os.path.join(location, slug), log = log) - if downloaded != root_dir: - raise ValueError('The location of mingw did not match\n%s\n%s' - % (downloaded, root_dir)) - - return root_dir - -def str2ver(string): - ''' - Converts a version string into a tuple - ''' - try: - version = tuple(int(v) for v in string.split('.')) - if len(version) is not 3: - raise ValueError() - except ValueError: - raise argparse.ArgumentTypeError( - 'please provide a three digit version string') - return version - -def main(): - ''' - Invoked when the script is run directly by the python interpreter - ''' - parser = argparse.ArgumentParser( - description = 'Downloads a specific version of MinGW', - formatter_class = argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('--location', - help = 'the location to download the compiler to', - default = os.path.join(tempfile.gettempdir(), 'mingw-builds')) - parser.add_argument('--arch', required = True, choices = ['i686', 'x86_64'], - help = 'the target MinGW architecture string') - parser.add_argument('--version', type = str2ver, - help = 'the version of GCC to download') - parser.add_argument('--threading', choices = ['posix', 'win32'], - help = 'the threading type of the compiler') - parser.add_argument('--exceptions', choices = ['sjlj', 'seh', 'dwarf'], - help = 'the method to throw exceptions') - parser.add_argument('--revision', type=int, - help = 'the revision of the MinGW release') - group = parser.add_mutually_exclusive_group() - group.add_argument('-v', '--verbose', action='store_true', - help='increase the script output verbosity') - group.add_argument('-q', '--quiet', action='store_true', - help='only print errors and warning') - args = parser.parse_args() - - # Create the logger - logger = logging.getLogger('mingw') - handler = logging.StreamHandler() - formatter = logging.Formatter('%(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - if args.quiet: - logger.setLevel(logging.WARN) - if args.verbose: - logger.setLevel(logging.DEBUG) - - # Get MinGW - root_dir = root(location = args.location, arch = args.arch, - version = args.version, threading = args.threading, - exceptions = args.exceptions, revision = args.revision, - log = logger) - - sys.stdout.write('%s\n' % os.path.join(root_dir, 'bin')) - -if __name__ == '__main__': - try: - main() - except IOError as e: - sys.stderr.write('IO error: %s\n' % e) - sys.exit(1) - except OSError as e: - sys.stderr.write('OS error: %s\n' % e) - sys.exit(1) - except KeyboardInterrupt as e: - sys.stderr.write('Killed\n') - sys.exit(1) From 6746c65bcfa49110bfe6642b8a47735637817be4 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Thu, 28 May 2020 09:33:06 +0100 Subject: [PATCH 212/330] Expose `SkipWithError` in Python bindings. (#968) --- bindings/python/benchmark/benchmark.cc | 3 ++- bindings/python/benchmark/example.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bindings/python/benchmark/benchmark.cc b/bindings/python/benchmark/benchmark.cc index ef95559659..374bf5479c 100644 --- a/bindings/python/benchmark/benchmark.cc +++ b/bindings/python/benchmark/benchmark.cc @@ -42,6 +42,7 @@ PYBIND11_MODULE(_benchmark, m) { py::class_(m, "State") .def("__bool__", &benchmark::State::KeepRunning) - .def_property_readonly("keep_running", &benchmark::State::KeepRunning); + .def_property_readonly("keep_running", &benchmark::State::KeepRunning) + .def("skip_with_error", &benchmark::State::SkipWithError); }; } // namespace diff --git a/bindings/python/benchmark/example.py b/bindings/python/benchmark/example.py index 24da12780a..9546699f92 100644 --- a/bindings/python/benchmark/example.py +++ b/bindings/python/benchmark/example.py @@ -28,5 +28,14 @@ def sum_million(state): sum(range(1_000_000)) +@benchmark.register +def skipped(state): + if True: # Test some predicate here. + state.skip_with_error('some error') + return # NOTE: You must explicitly return, or benchmark will continue. + + ... # Benchmark code would be here. + + if __name__ == '__main__': benchmark.main() From 4e88f582a908c69a40cc56d33c4ed6484f01ba24 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 5 Jun 2020 09:40:34 +0100 Subject: [PATCH 213/330] move releasing doc to docs --- releasing.md => docs/releasing.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename releasing.md => docs/releasing.md (100%) diff --git a/releasing.md b/docs/releasing.md similarity index 100% rename from releasing.md rename to docs/releasing.md From b63da7b84e40419da9194e2a312c426360aff293 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 8 Jun 2020 15:20:36 +0100 Subject: [PATCH 214/330] Ignore python bindings build artifacts --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a7716e3d5d..be55d774e2 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ CMakeSettings.json # Visual Studio Code cache/options directory .vscode/ + +# Python build stuff +dist/ +*.egg-info* From 74b4612c738f7cbccb8c4ed8060abb63a3e0c6a8 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 8 Jun 2020 15:21:52 +0100 Subject: [PATCH 215/330] Add absl dependency as needed --- WORKSPACE | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index dc6ea02b95..5438ad3611 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -8,6 +8,13 @@ http_archive( urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"], ) +http_archive( + name = "com_google_absl", + sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", + strip_prefix = "abseil-cpp-20200225.2", + urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], +) + http_archive( name = "com_google_googletest", strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", From 22e55e18eb2c52df2936f9357b6d615a4497bb35 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Mon, 8 Jun 2020 17:29:43 +0100 Subject: [PATCH 216/330] Add some simple docs for installing the Python bindings from source. (#975) --- bindings/python/benchmark/example.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bindings/python/benchmark/example.py b/bindings/python/benchmark/example.py index 9546699f92..7c8e3064ef 100644 --- a/bindings/python/benchmark/example.py +++ b/bindings/python/benchmark/example.py @@ -11,7 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Example of Python using C++ benchmark framework.""" +"""Example of Python using C++ benchmark framework. + +To run this example, you must first install the `benchmark` Python package. + +To install using `setup.py`, download and extract the `benchmark` source. +In the extracted directory, execute: + python setup.py install +""" import benchmark From 8039b4030795b1c9b8cedb78e3a2a6fb89574b6e Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 9 Jun 2020 09:50:20 +0100 Subject: [PATCH 217/330] Upgrade bazel from 0.10.1 to 3.2.0 (#976) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f220a7df61..36e343dbfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -211,11 +211,11 @@ install: - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo apt-get update -qq; sudo apt-get install -qq unzip cmake3; - wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh; + wget https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-linux-x86_64.sh --output-document bazel-installer.sh; travis_wait sudo bash bazel-installer.sh; fi - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then - curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-darwin-x86_64.sh; + curl -L -o bazel-installer.sh https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-darwin-x86_64.sh; travis_wait sudo bash bazel-installer.sh; fi From 99c52f141472a87c3016bb479fa3ef0e67789950 Mon Sep 17 00:00:00 2001 From: Brian Wolfe Date: Mon, 15 Jun 2020 09:28:17 -0700 Subject: [PATCH 218/330] use rfc3339-formatted timestamps in output [output format change] (#965) * timestamp: use rfc3339-formatted timestamps in output Replace localized timestamps with machine-readable IETF RFC 3339 format timestamps. This is an attempt to make the output timestamps easily machine-readable. ISO8601 specifies standards for time interchange formats. IETF RFC 3339: https://tools.ietf.org/html/rfc3339 defines a subset of these for use in the internet. The general form for these timestamps is: YYYY-MM-DDTHH:mm:SS[+-]hhmm This replaces the localized time formats that are currently being used in the benchmark output to prioritize interchangeability and machine-readability. This might break existing programs that rely on the particular date-time format. This might also may make times less human readable. RFC3339 was intended to balance human readability and simplicity for machine readability, but it is primarily intended as an internal representation. * timers: remove utc string formatting We only ever need local time printing. Remove the UTC printing and cosnolidate the logic slightly. * timers: manually create rfc3339 string The C++ standard library does not output the time offset in RFC3339 format, it is missing the : between hours and minutes. VS does not appear to support timezone information by default. To avoid adding too much complexity to benchmark around timezone handling e.g. a full date library like https://github.com/HowardHinnant/date, we fall back to outputting GMT time with a -00:00 offset for those cases. * timers: use reentrant form for localtime_r & tmtime_r For non-windows, use the reentrant form for the time conversion functions. * timers: cleanup Use strtol instead of brittle moving characters around. * timers: only call strftime twice. Also size buffers to known maximum necessary size and name constants more appropriately. * timers: fix unused variable warning --- src/timers.cc | 66 ++++++++++++++++++++++++------------ test/reporter_output_test.cc | 2 +- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index 7613ff92c6..221b594a3a 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -178,40 +178,64 @@ double ThreadCPUUsage() { #endif } -namespace { - -std::string DateTimeString(bool local) { +std::string LocalDateTimeString() { + // Write the local time in RFC3339 format yyyy-mm-ddTHH:MM:SS+/-HH:MM. typedef std::chrono::system_clock Clock; std::time_t now = Clock::to_time_t(Clock::now()); - const std::size_t kStorageSize = 128; - char storage[kStorageSize]; - std::size_t written; + const std::size_t kTzOffsetLen = 6; + const std::size_t kTimestampLen = 19; + + std::size_t tz_len; + std::size_t timestamp_len; + long int offset_minutes; + char tz_offset_sign = '+'; + char tz_offset[kTzOffsetLen + 1]; + char storage[kTimestampLen + kTzOffsetLen + 1]; - if (local) { #if defined(BENCHMARK_OS_WINDOWS) - written = - std::strftime(storage, sizeof(storage), "%x %X", ::localtime(&now)); + std::tm *timeinfo_p = ::localtime(&now); #else - std::tm timeinfo; - ::localtime_r(&now, &timeinfo); - written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); + std::tm timeinfo; + std::tm *timeinfo_p = &timeinfo; + ::localtime_r(&now, &timeinfo); #endif + + tz_len = std::strftime(tz_offset, sizeof(tz_offset), "%z", timeinfo_p); + + if (tz_len < kTzOffsetLen && tz_len > 1) { + // Timezone offset was written. strftime writes offset as +HHMM or -HHMM, + // RFC3339 specifies an offset as +HH:MM or -HH:MM. To convert, we parse + // the offset as an integer, then reprint it to a string. + + offset_minutes = ::strtol(tz_offset, NULL, 10); + if (offset_minutes < 0) { + offset_minutes *= -1; + tz_offset_sign = '-'; + } + tz_len = ::sprintf(tz_offset, "%c%02li:%02li", tz_offset_sign, + offset_minutes / 100, offset_minutes % 100); + CHECK(tz_len == 6); + ((void)tz_len); // Prevent unused variable warning in optimized build. } else { + // Unknown offset. RFC3339 specifies that unknown local offsets should be + // written as UTC time with -00:00 timezone. #if defined(BENCHMARK_OS_WINDOWS) - written = std::strftime(storage, sizeof(storage), "%x %X", ::gmtime(&now)); + // Potential race condition if another thread calls localtime or gmtime. + timeinfo_p = ::gmtime(&now); #else - std::tm timeinfo; ::gmtime_r(&now, &timeinfo); - written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); #endif + + strncpy(tz_offset, "-00:00", kTzOffsetLen + 1); } - CHECK(written < kStorageSize); - ((void)written); // prevent unused variable in optimized mode. - return std::string(storage); -} -} // end namespace + timestamp_len = std::strftime(storage, sizeof(storage), "%Y-%m-%dT%H:%M:%S", + timeinfo_p); + CHECK(timestamp_len == kTimestampLen); + ((void)timestamp_len); // Prevent unused variable warning in optimized build. -std::string LocalDateTimeString() { return DateTimeString(true); } + std::strncat(storage, tz_offset, kTzOffsetLen + 1); + return std::string(storage); +} } // end namespace benchmark diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 1a96b5f0bf..d806a4e82a 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -15,7 +15,7 @@ ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, static int AddContextCases() { AddCases(TC_ConsoleErr, { - {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, + {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default}, {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); From f6ac240cd290ca0fb8f4205b6af55ecc59abbb7a Mon Sep 17 00:00:00 2001 From: Brian Wolfe Date: Mon, 15 Jun 2020 14:02:15 -0700 Subject: [PATCH 219/330] timers: silence format overflow warning --- src/timers.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/timers.cc b/src/timers.cc index 221b594a3a..2501bab9be 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -212,8 +212,10 @@ std::string LocalDateTimeString() { offset_minutes *= -1; tz_offset_sign = '-'; } + + // Apply % 100 to hour to guarantee range [0, 99]. tz_len = ::sprintf(tz_offset, "%c%02li:%02li", tz_offset_sign, - offset_minutes / 100, offset_minutes % 100); + (offset_minutes / 100) % 100, offset_minutes % 100); CHECK(tz_len == 6); ((void)tz_len); // Prevent unused variable warning in optimized build. } else { From f25ea40ae190fab5fcbea22abd3694de045701c7 Mon Sep 17 00:00:00 2001 From: Brian Wolfe Date: Mon, 15 Jun 2020 14:16:20 -0700 Subject: [PATCH 220/330] timers: use snprintf instead of sprintf --- src/timers.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index 2501bab9be..dbc7ba39e0 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -213,9 +213,8 @@ std::string LocalDateTimeString() { tz_offset_sign = '-'; } - // Apply % 100 to hour to guarantee range [0, 99]. - tz_len = ::sprintf(tz_offset, "%c%02li:%02li", tz_offset_sign, - (offset_minutes / 100) % 100, offset_minutes % 100); + tz_len = ::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li", + tz_offset_sign, (offset_minutes / 100), offset_minutes % 100); CHECK(tz_len == 6); ((void)tz_len); // Prevent unused variable warning in optimized build. } else { From 7cc06ef80ccdd9f90153203435f7e7b4b41370a5 Mon Sep 17 00:00:00 2001 From: Brian Wolfe Date: Mon, 15 Jun 2020 15:58:30 -0700 Subject: [PATCH 221/330] timers: just make the buffers big enough --- src/timers.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index dbc7ba39e0..b95024432b 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -189,8 +189,9 @@ std::string LocalDateTimeString() { std::size_t timestamp_len; long int offset_minutes; char tz_offset_sign = '+'; - char tz_offset[kTzOffsetLen + 1]; - char storage[kTimestampLen + kTzOffsetLen + 1]; + // Long enough buffers to avoid format-overflow warnings + char tz_offset[128]; + char storage[128]; #if defined(BENCHMARK_OS_WINDOWS) std::tm *timeinfo_p = ::localtime(&now); @@ -214,8 +215,8 @@ std::string LocalDateTimeString() { } tz_len = ::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li", - tz_offset_sign, (offset_minutes / 100), offset_minutes % 100); - CHECK(tz_len == 6); + tz_offset_sign, offset_minutes / 100, offset_minutes % 100); + CHECK(tz_len == kTzOffsetLen); ((void)tz_len); // Prevent unused variable warning in optimized build. } else { // Unknown offset. RFC3339 specifies that unknown local offsets should be @@ -233,7 +234,9 @@ std::string LocalDateTimeString() { timestamp_len = std::strftime(storage, sizeof(storage), "%Y-%m-%dT%H:%M:%S", timeinfo_p); CHECK(timestamp_len == kTimestampLen); - ((void)timestamp_len); // Prevent unused variable warning in optimized build. + // Prevent unused variable warning in optimized build. + ((void)timestamp_len); + ((void)kTimestampLen); std::strncat(storage, tz_offset, kTzOffsetLen + 1); return std::string(storage); From 15e6dfd7182b74b4fb6860f52fe314d0654307fb Mon Sep 17 00:00:00 2001 From: Reid Paape Date: Wed, 17 Jun 2020 06:58:12 -0500 Subject: [PATCH 222/330] timers: silence strncat truncation warning (#984) --- src/timers.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index b95024432b..4f76eddc1d 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -235,10 +235,9 @@ std::string LocalDateTimeString() { timeinfo_p); CHECK(timestamp_len == kTimestampLen); // Prevent unused variable warning in optimized build. - ((void)timestamp_len); ((void)kTimestampLen); - std::strncat(storage, tz_offset, kTzOffsetLen + 1); + std::strncat(storage, tz_offset, sizeof(storage) - timestamp_len - 1); return std::string(storage); } From 39b6e703f8cdf87284db2aaca2f9b214f02e5673 Mon Sep 17 00:00:00 2001 From: Jonas Otto Date: Thu, 25 Jun 2020 12:46:13 +0200 Subject: [PATCH 223/330] adds a "--no-color" flag to the compare.py script (#990) --- tools/compare.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/compare.py b/tools/compare.py index 539ace6fb1..bd01be57cd 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -48,6 +48,14 @@ def create_parser(): "of repetitions. Do note that only the display is affected. " "Internally, all the actual runs are still used, e.g. for U test.") + parser.add_argument( + '--no-color', + dest='color', + default=True, + action="store_false", + help="Do not use colors in the terminal output" + ) + utest = parser.add_argument_group() utest.add_argument( '--no-utest', @@ -239,7 +247,7 @@ def main(): # Diff and output output_lines = gbench.report.generate_difference_report( json1, json2, args.display_aggregates_only, - args.utest, args.utest_alpha) + args.utest, args.utest_alpha, args.color) print(description) for ln in output_lines: print(ln) From 8f5e6ae091867060a0ce82fd2675adde05557ac9 Mon Sep 17 00:00:00 2001 From: Jonas Otto Date: Mon, 29 Jun 2020 14:22:03 +0200 Subject: [PATCH 224/330] Add requirements.txt for python dependencies (#994) * add requirements.txt for python tools * adds documentation for requirements.txt Adds installation instructions for python dependencies using pip and requirements.txt --- docs/tools.md | 6 +++++- tools/requirements.txt | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tools/requirements.txt diff --git a/docs/tools.md b/docs/tools.md index 4a3b2e9bd2..f2d0c497f3 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -4,7 +4,11 @@ The `compare.py` can be used to compare the result of benchmarks. -**NOTE**: the utility relies on the scipy package which can be installed using [these instructions](https://www.scipy.org/install.html). +### Dependencies +The utility relies on the [scipy](https://www.scipy.org) package which can be installed using pip: +```bash +pip3 install -r requirements.txt +``` ### Displaying aggregates only diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000000..3b3331b5af --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1 @@ +scipy>=1.5.0 \ No newline at end of file From 7f27afe83b82f3a98baf58ef595814b9d42a5b2b Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Tue, 30 Jun 2020 01:51:30 -0700 Subject: [PATCH 225/330] Expose methods for custom main functions in Python. (#993) --- bindings/python/benchmark/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bindings/python/benchmark/__init__.py b/bindings/python/benchmark/__init__.py index 27f76e0567..378f030455 100644 --- a/bindings/python/benchmark/__init__.py +++ b/bindings/python/benchmark/__init__.py @@ -60,3 +60,8 @@ def _run_benchmarks(argv): def main(argv=None): return app.run(_run_benchmarks, argv=argv, flags_parser=_flags_parser) + + +# Methods for use with custom main function. +initialize = _benchmark.Initialize +run_benchmarks = _benchmark.RunSpecifiedBenchmarks From 39c8d58a76ce7c3398786ee96a452668e8392af7 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Thu, 9 Jul 2020 09:23:06 +0100 Subject: [PATCH 226/330] Rename python bindings package to `google_benchmark`. (#999) A few people have complained that `benchmark` is too generic. Also, add Python 3.8. --- bindings/python/{benchmark => google_benchmark}/BUILD | 4 ++-- .../python/{benchmark => google_benchmark}/__init__.py | 0 .../python/{benchmark => google_benchmark}/benchmark.cc | 0 bindings/python/{benchmark => google_benchmark}/example.py | 6 +++--- setup.py | 7 ++++--- 5 files changed, 9 insertions(+), 8 deletions(-) rename bindings/python/{benchmark => google_benchmark}/BUILD (91%) rename bindings/python/{benchmark => google_benchmark}/__init__.py (100%) rename bindings/python/{benchmark => google_benchmark}/benchmark.cc (100%) rename bindings/python/{benchmark => google_benchmark}/example.py (85%) diff --git a/bindings/python/benchmark/BUILD b/bindings/python/google_benchmark/BUILD similarity index 91% rename from bindings/python/benchmark/BUILD rename to bindings/python/google_benchmark/BUILD index 49f536e615..3c1561f48e 100644 --- a/bindings/python/benchmark/BUILD +++ b/bindings/python/google_benchmark/BUILD @@ -1,7 +1,7 @@ load("//bindings/python:build_defs.bzl", "py_extension") py_library( - name = "benchmark", + name = "google_benchmark", srcs = ["__init__.py"], visibility = ["//visibility:public"], deps = [ @@ -32,7 +32,7 @@ py_test( srcs_version = "PY3", visibility = ["//visibility:public"], deps = [ - ":benchmark", + ":google_benchmark", ], ) diff --git a/bindings/python/benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py similarity index 100% rename from bindings/python/benchmark/__init__.py rename to bindings/python/google_benchmark/__init__.py diff --git a/bindings/python/benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc similarity index 100% rename from bindings/python/benchmark/benchmark.cc rename to bindings/python/google_benchmark/benchmark.cc diff --git a/bindings/python/benchmark/example.py b/bindings/python/google_benchmark/example.py similarity index 85% rename from bindings/python/benchmark/example.py rename to bindings/python/google_benchmark/example.py index 7c8e3064ef..e968462479 100644 --- a/bindings/python/benchmark/example.py +++ b/bindings/python/google_benchmark/example.py @@ -13,14 +13,14 @@ # limitations under the License. """Example of Python using C++ benchmark framework. -To run this example, you must first install the `benchmark` Python package. +To run this example, you must first install the `google_benchmark` Python package. -To install using `setup.py`, download and extract the `benchmark` source. +To install using `setup.py`, download and extract the `google_benchmark` source. In the extracted directory, execute: python setup.py install """ -import benchmark +import google_benchmark as benchmark @benchmark.register diff --git a/setup.py b/setup.py index f4c063346a..a2b0b917ad 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def _get_version(): """Parse the version string from __init__.py.""" - with open(os.path.join(here, 'bindings', 'python', 'benchmark', '__init__.py')) as f: + with open(os.path.join(here, 'bindings', 'python', 'google_benchmark', '__init__.py')) as f: try: version_line = next( line for line in f if line.startswith('__version__')) @@ -95,7 +95,7 @@ def bazel_build(self, ext): setuptools.setup( - name='google-benchmark', + name='google_benchmark', version=_get_version(), url='https://github.com/google/benchmark', description='A library to benchmark code snippets.', @@ -106,7 +106,7 @@ def bazel_build(self, ext): packages=setuptools.find_packages('bindings/python'), install_requires=_parse_requirements('bindings/python/requirements.txt'), cmdclass=dict(build_ext=BuildBazelExtension), - ext_modules=[BazelExtension('benchmark._benchmark', '//bindings/python/benchmark:_benchmark')], + ext_modules=[BazelExtension('google_benchmark._benchmark', '//bindings/python/google_benchmark:_benchmark')], zip_safe=False, # PyPI package information. classifiers=[ @@ -116,6 +116,7 @@ def bazel_build(self, ext): 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Software Development :: Testing', 'Topic :: System :: Benchmark', ], From 37177a84b7e8d33696ea1e1854513cb0de3b4dc3 Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Thu, 9 Jul 2020 12:54:41 +0100 Subject: [PATCH 227/330] Fix python extension import (#1000) --- bindings/python/google_benchmark/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index 378f030455..c3a93bfc23 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -14,7 +14,7 @@ """Python benchmarking utilities. Example usage: - import benchmark + import google_benchmark as benchmark @benchmark.register def my_benchmark(state): @@ -28,7 +28,7 @@ def my_benchmark(state): """ from absl import app -from benchmark import _benchmark +from google_benchmark import _benchmark __all__ = [ "register", From 99010118805eb12ce56122f859c3604c2a427d05 Mon Sep 17 00:00:00 2001 From: Alexander Enaldiev Date: Tue, 28 Jul 2020 14:46:07 +0300 Subject: [PATCH 228/330] JSONReporter: don't report on scaling if we didn't get it (#1005) (#1008) * JSONReporter: don't report on scaling if we didn't get it (#1005) * JSONReporter: fix due to review (std::pair -> enum) * JSONReporter: scaling: fix the algo (due to review discussion) * benchmark.h: revert to old-fashioned enum's (C++03 compatibility); rreporter_output_test: let's skip scaling --- include/benchmark/benchmark.h | 9 ++++++++- src/json_reporter.cc | 6 ++++-- src/reporter.cc | 2 +- src/sysinfo.cc | 14 ++++++++------ test/reporter_output_test.cc | 3 +-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index e5f6778958..da638f952d 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -176,6 +176,7 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include #include +#include #include #if defined(BENCHMARK_HAS_CXX11) @@ -1294,10 +1295,16 @@ struct CPUInfo { int num_sharing; }; + enum Scaling { + UNKNOWN, + ENABLED, + DISABLED + }; + int num_cpus; double cycles_per_second; std::vector caches; - bool scaling_enabled; + Scaling scaling; std::vector load_avg; static const CPUInfo& Get(); diff --git a/src/json_reporter.cc b/src/json_reporter.cc index e5f3c35248..959d245a34 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -122,8 +122,10 @@ bool JSONReporter::ReportContext(const Context& context) { << FormatKV("mhz_per_cpu", RoundDouble(info.cycles_per_second / 1000000.0)) << ",\n"; - out << indent << FormatKV("cpu_scaling_enabled", info.scaling_enabled) - << ",\n"; + if (CPUInfo::Scaling::UNKNOWN != info.scaling) { + out << indent << FormatKV("cpu_scaling_enabled", info.scaling == CPUInfo::Scaling::ENABLED ? true : false) + << ",\n"; + } out << indent << "\"caches\": [\n"; indent = std::string(6, ' '); diff --git a/src/reporter.cc b/src/reporter.cc index 0b54fa421a..337575a118 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -64,7 +64,7 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } - if (info.scaling_enabled) { + if (CPUInfo::Scaling::ENABLED == info.scaling) { Out << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " "overhead.\n"; diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 5b7c4af780..3947f959f9 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -57,6 +57,7 @@ #include #include #include +#include #include "check.h" #include "cycleclock.h" @@ -209,11 +210,11 @@ bool ReadFromFile(std::string const& fname, ArgT* arg) { return f.good(); } -bool CpuScalingEnabled(int num_cpus) { +CPUInfo::Scaling CpuScaling(int num_cpus) { // We don't have a valid CPU count, so don't even bother. - if (num_cpus <= 0) return false; + if (num_cpus <= 0) return CPUInfo::Scaling::UNKNOWN; #ifdef BENCHMARK_OS_QNX - return false; + return CPUInfo::Scaling::UNKNOWN; #endif #ifndef BENCHMARK_OS_WINDOWS // On Linux, the CPUfreq subsystem exposes CPU information as files on the @@ -223,10 +224,11 @@ bool CpuScalingEnabled(int num_cpus) { for (int cpu = 0; cpu < num_cpus; ++cpu) { std::string governor_file = StrCat("/sys/devices/system/cpu/cpu", cpu, "/cpufreq/scaling_governor"); - if (ReadFromFile(governor_file, &res) && res != "performance") return true; + if (ReadFromFile(governor_file, &res) && res != "performance") return CPUInfo::Scaling::ENABLED; } + return CPUInfo::Scaling::DISABLED; #endif - return false; + return CPUInfo::Scaling::UNKNOWN; } int CountSetBitsInCPUMap(std::string Val) { @@ -695,7 +697,7 @@ CPUInfo::CPUInfo() : num_cpus(GetNumCPUs()), cycles_per_second(GetCPUCyclesPerSecond()), caches(GetCacheSizes()), - scaling_enabled(CpuScalingEnabled(num_cpus)), + scaling(CpuScaling(num_cpus)), load_avg(GetLoadAvg()) {} diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index d806a4e82a..bcce007831 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -28,8 +28,7 @@ static int AddContextCases() { MR_Next}, {"\"num_cpus\": %int,$", MR_Next}, {"\"mhz_per_cpu\": %float,$", MR_Next}, - {"\"cpu_scaling_enabled\": ", MR_Next}, - {"\"caches\": \\[$", MR_Next}}); + {"\"caches\": \\[$", MR_Default}}); auto const& Info = benchmark::CPUInfo::Get(); auto const& Caches = Info.caches; if (!Caches.empty()) { From 1302d2ce094a9753b0f81a81ea74c0fa71fae582 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 30 Jul 2020 09:51:48 +0100 Subject: [PATCH 229/330] Add missing breaks for QNX cache counting (#1012) --- src/sysinfo.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 3947f959f9..8bab9320f1 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -384,9 +384,11 @@ std::vector GetCacheSizesQNX() { case CACHE_FLAG_UNIFIED : info.type = "Unified"; info.level = 2; + break; case CACHE_FLAG_SHARED : info.type = "Shared"; info.level = 3; + break; default : continue; break; From 4986d0b2ea36909ecc5873b1d740fdca1f7a93a0 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 18 Aug 2020 08:53:21 +0100 Subject: [PATCH 230/330] Create build-and-test.yml (#1013) First attempt at a non-travis/non appveyor CI solution --- .github/workflows/build-and-test.yml | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/build-and-test.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000000..599b0943d0 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,38 @@ +name: build-and-test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + job: + # TODO(dominic): Extend this to include compiler and set through env: CC/CXX. + name: ${{ matrix.os }}.${{ matrix.build_type }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-16.04, ubuntu-20.04, macos-latest, windows-latest] + build_type: ['Release', 'Debug'] + steps: + - uses: actions/checkout@v2 + + - name: create build environment + run: cmake -E make_directory ${{ runner.workspace }}/_build + + - name: configure cmake + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + + - name: build + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} + + - name: test + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: ctest -C ${{ matrix.build_type }} From 5b72b6c2da65472e1de7e9180e116c1e5beadb20 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 18 Aug 2020 10:02:20 +0100 Subject: [PATCH 231/330] Remove "BENCHMARK_" prefix from env var version of command line flags (#997) As noted in #995, this causes issues when the command line flag already starts with "benchmark_", which they all do. Not caught by tests as the test flags didn't start with "benchmark". Fixes #995 --- src/commandlineflags.cc | 2 +- test/commandlineflags_gtest.cc | 144 ++++++++++++++++----------------- 2 files changed, 73 insertions(+), 73 deletions(-) diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 3380a127a8..0648fe3a06 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -88,7 +88,7 @@ static std::string FlagToEnvVar(const char* flag) { for (size_t i = 0; i != flag_str.length(); ++i) env_var += static_cast(::toupper(flag_str.c_str()[i])); - return "BENCHMARK_" + env_var; + return env_var; } } // namespace diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc index 36bdb44595..656020f2ec 100644 --- a/test/commandlineflags_gtest.cc +++ b/test/commandlineflags_gtest.cc @@ -26,175 +26,175 @@ int unsetenv(const char* name) { #endif // BENCHMARK_OS_WINDOWS TEST(BoolFromEnv, Default) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); EXPECT_EQ(BoolFromEnv("not_in_env", true), true); } TEST(BoolFromEnv, False) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "0", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "N", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "N", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "n", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "n", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "NO", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "NO", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "No", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "No", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "no", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "no", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "F", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "F", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "f", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "f", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "FALSE", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "FALSE", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "False", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "False", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "false", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "false", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "OFF", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "OFF", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Off", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "Off", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "off", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "off", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", true), false); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } TEST(BoolFromEnv, True) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "1", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "1", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Y", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "Y", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "y", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "y", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "YES", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "YES", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Yes", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "Yes", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "yes", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "yes", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "T", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "T", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "t", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "t", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "TRUE", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "TRUE", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "True", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "True", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "true", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "true", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "ON", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "ON", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "On", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "On", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "on", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "on", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); #ifndef BENCHMARK_OS_WINDOWS - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "", 1), 0); EXPECT_EQ(BoolFromEnv("in_env", false), true); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); #endif } TEST(Int32FromEnv, NotInEnv) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); EXPECT_EQ(Int32FromEnv("not_in_env", 42), 42); } TEST(Int32FromEnv, InvalidInteger) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0); EXPECT_EQ(Int32FromEnv("in_env", 42), 42); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } TEST(Int32FromEnv, ValidInteger) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "42", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "42", 1), 0); EXPECT_EQ(Int32FromEnv("in_env", 64), 42); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } TEST(DoubleFromEnv, NotInEnv) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); EXPECT_EQ(DoubleFromEnv("not_in_env", 0.51), 0.51); } TEST(DoubleFromEnv, InvalidReal) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0); EXPECT_EQ(DoubleFromEnv("in_env", 0.51), 0.51); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } TEST(DoubleFromEnv, ValidReal) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0.51", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "0.51", 1), 0); EXPECT_EQ(DoubleFromEnv("in_env", 0.71), 0.51); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } TEST(StringFromEnv, Default) { - ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0); + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); EXPECT_STREQ(StringFromEnv("not_in_env", "foo"), "foo"); } TEST(StringFromEnv, Valid) { - ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0); + ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0); EXPECT_STREQ(StringFromEnv("in_env", "bar"), "foo"); - unsetenv("BENCHMARK_IN_ENV"); + unsetenv("IN_ENV"); } } // namespace From bb978c06d0fc326c8e87cf6ef4fbbb44ab1704db Mon Sep 17 00:00:00 2001 From: Adam Badura Date: Wed, 19 Aug 2020 12:57:19 +0200 Subject: [PATCH 232/330] Update build instructions to better use CMake (#1017) Build instructions needlessly referred to make when CMake offers a command-line interface to abstract away from the specific build system. Furthermore, CMake offers command-line "tool mode" which performs basic filesystem operations. While the syntax is a bit more verbose than Linux commands it is platform-independent. Now the commands can be copy-pasted on both Linux and Windows and will just work. Finally, the Release build type is included in initial commands. A natural flow for a new-comer is to read and execute the commands and only then learn that one has to go back and redo them again this time with proper parameters. Now instead the parameters are only explained later but present already in the initial commands. --- README.md | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 02a3bfad80..1f3d3896d5 100644 --- a/README.md +++ b/README.md @@ -70,13 +70,13 @@ $ git clone https://github.com/google/googletest.git benchmark/googletest # Go to the library root directory $ cd benchmark # Make a build directory to place the build output. -$ mkdir build && cd build -# Generate a Makefile with cmake. -# Use cmake -G to generate a different file type. -$ cmake ../ +$ cmake -E make_directory "build" +# Generate build system files with cmake. +$ cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release ../ +# or, starting with CMake 3.13, use a simpler form: +# cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build" # Build the library. -# Use make -j to speed up the build process, e.g. make -j8 . -$ make +$ cmake --build "build" --config Release --parallel ``` This builds the `benchmark` and `benchmark_main` libraries and tests. On a unix system, the build directory should now look something like this: @@ -94,13 +94,13 @@ On a unix system, the build directory should now look something like this: Next, you can run the tests to check the build. ```bash -$ make test +$ cmake --build "build" --config Release --target test ``` If you want to install the library globally, also run: ``` -sudo make install +sudo cmake --build "build" --config Release --target install ``` Note that Google Benchmark requires Google Test to build and run the tests. This @@ -117,17 +117,14 @@ to `CMAKE_ARGS`. ### Debug vs Release By default, benchmark builds as a debug library. You will see a warning in the -output when this is the case. To build it as a release library instead, use: - -``` -cmake -DCMAKE_BUILD_TYPE=Release -``` - -To enable link-time optimisation, use - -``` -cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true -``` +output when this is the case. To build it as a release library instead, add +`-DCMAKE_BUILD_TYPE=Release` when generating the build system files, as shown +above. The use of `--config Release` in build commands is needed to properly +support multi-configuration tools (like Visual Studio for example) and can be +skipped for other build systems (like Makefile). + +To enable link-time optimisation, also add `-DBENCHMARK_ENABLE_LTO=true` when +generating the build system files. If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake cache variables, if autodetection fails. From 5c25ad3acb24802943935783aadd5e035e6edbf0 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 21 Aug 2020 16:25:56 +0100 Subject: [PATCH 233/330] Ctest support (#1025) * ctest is now working * Update README * remove commented out lines * Tweaked docs Added note to use parallel and cleaned build config notes * Response to comments * revert all but the readme * make error message clearer * drop --parallel --- README.md | 5 ++--- cmake/GoogleTest.cmake.in | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1f3d3896d5..314feee068 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ $ cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release ../ # or, starting with CMake 3.13, use a simpler form: # cmake -DCMAKE_BUILD_TYPE=Release -S . -B "build" # Build the library. -$ cmake --build "build" --config Release --parallel +$ cmake --build "build" --config Release ``` This builds the `benchmark` and `benchmark_main` libraries and tests. On a unix system, the build directory should now look something like this: @@ -94,7 +94,7 @@ On a unix system, the build directory should now look something like this: Next, you can run the tests to check the build. ```bash -$ cmake --build "build" --config Release --target test +$ cmake -E chdir "build" ctest --build-config Release ``` If you want to install the library globally, also run: @@ -132,7 +132,6 @@ cache variables, if autodetection fails. If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. - ### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; diff --git a/cmake/GoogleTest.cmake.in b/cmake/GoogleTest.cmake.in index 28818ee293..fd957ff564 100644 --- a/cmake/GoogleTest.cmake.in +++ b/cmake/GoogleTest.cmake.in @@ -31,7 +31,7 @@ if(EXISTS "${GOOGLETEST_PATH}" AND IS_DIRECTORY "${GOOGLETEST_PATH}" ) else() if(NOT ALLOW_DOWNLOADING_GOOGLETEST) - message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable ALLOW_DOWNLOADING_GOOGLETEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") + message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable BENCHMARK_DOWNLOAD_DEPENDENCIES, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.") else() message(WARNING "Did not find Google Test sources! Fetching from web...") ExternalProject_Add( From 4857962394266165790de2266a695f328fc144f3 Mon Sep 17 00:00:00 2001 From: Christian Wassermann Date: Tue, 25 Aug 2020 14:47:44 +0200 Subject: [PATCH 234/330] Add CartesianProduct with associated test (#1029) * Add CartesianProduct with associated test * Use CartesianProduct in Ranges to avoid code duplication * Add new cartesian_product_test to CMakeLists.txt * Update AUTHORS & CONTRIBUTORS * Rename CartesianProduct to ArgsProduct * Rename test & fixture accordingly * Add example for ArgsProduct to README --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 23 +++++++++++ include/benchmark/benchmark.h | 5 +++ src/benchmark_register.cc | 43 +++++++++++-------- test/CMakeLists.txt | 3 ++ test/args_product_test.cc | 77 +++++++++++++++++++++++++++++++++++ 7 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 test/args_product_test.cc diff --git a/AUTHORS b/AUTHORS index 89205a1adb..e353b53bf3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Alex Steele Andriy Berestovskyy Arne Beer Carto +Christian Wassermann Christopher Seymour Colin Braley Daniel Harvey diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 88f7eee06c..6beed7166e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -28,6 +28,7 @@ Andriy Berestovskyy Arne Beer Billy Robert O'Neal III Chris Kennelly +Christian Wassermann Christopher Seymour Colin Braley Cyrille Faucheux diff --git a/README.md b/README.md index 314feee068..41a1bdff75 100644 --- a/README.md +++ b/README.md @@ -548,6 +548,29 @@ pair. BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` +Some benchmarks may require specific argument values that cannot be expressed +with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a +benchmark input for each combination in the product of the supplied vectors. + +```c++ +BENCHMARK(BM_SetInsert) + ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) +// would generate the same benchmark arguments as +BENCHMARK(BM_SetInsert) + ->Args({1<<10, 20}) + ->Args({3<<10, 20}) + ->Args({8<<10, 20}) + ->Args({3<<10, 40}) + ->Args({8<<10, 40}) + ->Args({1<<10, 40}) + ->Args({1<<10, 60}) + ->Args({3<<10, 60}) + ->Args({8<<10, 60}) + ->Args({1<<10, 80}) + ->Args({3<<10, 80}) + ->Args({8<<10, 80}); +``` + For more complex patterns of inputs, passing a custom function to `Apply` allows programmatic specification of an arbitrary set of arguments on which to run the benchmark. The following example enumerates a dense range on one parameter, diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index da638f952d..01f12620ee 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -828,6 +828,11 @@ class Benchmark { // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... Benchmark* Ranges(const std::vector >& ranges); + // Run this benchmark once for each combination of values in the (cartesian) + // product of the supplied argument lists. + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* ArgsProduct(const std::vector >& arglists); + // Equivalent to ArgNames({name}) Benchmark* ArgName(const std::string& name); diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index cca39b2215..65d9944f4f 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -303,33 +304,41 @@ Benchmark* Benchmark::Ranges( const std::vector>& ranges) { CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); std::vector> arglists(ranges.size()); - std::size_t total = 1; for (std::size_t i = 0; i < ranges.size(); i++) { AddRange(&arglists[i], ranges[i].first, ranges[i].second, range_multiplier_); - total *= arglists[i].size(); } - std::vector ctr(arglists.size(), 0); + ArgsProduct(arglists); - for (std::size_t i = 0; i < total; i++) { - std::vector tmp; - tmp.reserve(arglists.size()); - - for (std::size_t j = 0; j < arglists.size(); j++) { - tmp.push_back(arglists[j].at(ctr[j])); - } + return this; +} - args_.push_back(std::move(tmp)); +Benchmark* Benchmark::ArgsProduct( + const std::vector>& arglists) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(arglists.size())); - for (std::size_t j = 0; j < arglists.size(); j++) { - if (ctr[j] + 1 < arglists[j].size()) { - ++ctr[j]; - break; - } - ctr[j] = 0; + std::vector indices(arglists.size()); + const std::size_t total = std::accumulate( + std::begin(arglists), std::end(arglists), std::size_t{1}, + [](const std::size_t res, const std::vector& arglist) { + return res * arglist.size(); + }); + std::vector args; + args.reserve(arglists.size()); + for (std::size_t i = 0; i < total; i++) { + for (std::size_t arg = 0; arg < arglists.size(); arg++) { + args.push_back(arglists[arg][indices[arg]]); } + args_.push_back(args); + args.clear(); + + std::size_t arg = 0; + do { + indices[arg] = (indices[arg] + 1) % arglists[arg].size(); + } while (indices[arg++] == 0 && arg < arglists.size()); } + return this; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0d228b85cb..c1a3a3fc19 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -113,6 +113,9 @@ add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) compile_benchmark_test(multiple_ranges_test) add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +compile_benchmark_test(args_product_test) +add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) + compile_benchmark_test_with_main(link_main_test) add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) diff --git a/test/args_product_test.cc b/test/args_product_test.cc new file mode 100644 index 0000000000..8a859f8415 --- /dev/null +++ b/test/args_product_test.cc @@ -0,0 +1,77 @@ +#include "benchmark/benchmark.h" + +#include +#include +#include +#include + +class ArgsProductFixture : public ::benchmark::Fixture { + public: + ArgsProductFixture() + : expectedValues({{0, 100, 2000, 30000}, + {1, 15, 3, 8}, + {1, 15, 3, 9}, + {1, 15, 7, 8}, + {1, 15, 7, 9}, + {1, 15, 10, 8}, + {1, 15, 10, 9}, + {2, 15, 3, 8}, + {2, 15, 3, 9}, + {2, 15, 7, 8}, + {2, 15, 7, 9}, + {2, 15, 10, 8}, + {2, 15, 10, 9}, + {4, 5, 6, 11}}) {} + + void SetUp(const ::benchmark::State& state) { + std::vector ranges = {state.range(0), state.range(1), + state.range(2), state.range(3)}; + + assert(expectedValues.find(ranges) != expectedValues.end()); + + actualValues.insert(ranges); + } + + // NOTE: This is not TearDown as we want to check after _all_ runs are + // complete. + virtual ~ArgsProductFixture() { + if (actualValues != expectedValues) { + std::cout << "EXPECTED\n"; + for (auto v : expectedValues) { + std::cout << "{"; + for (int64_t iv : v) { + std::cout << iv << ", "; + } + std::cout << "}\n"; + } + std::cout << "ACTUAL\n"; + for (auto v : actualValues) { + std::cout << "{"; + for (int64_t iv : v) { + std::cout << iv << ", "; + } + std::cout << "}\n"; + } + } + } + + std::set> expectedValues; + std::set> actualValues; +}; + +BENCHMARK_DEFINE_F(ArgsProductFixture, Empty)(benchmark::State& state) { + for (auto _ : state) { + int64_t product = + state.range(0) * state.range(1) * state.range(2) * state.range(3); + for (int64_t x = 0; x < product; x++) { + benchmark::DoNotOptimize(x); + } + } +} + +BENCHMARK_REGISTER_F(ArgsProductFixture, Empty) + ->Args({0, 100, 2000, 30000}) + ->ArgsProduct({{1, 2}, {15}, {3, 7, 10}, {8, 9}}) + ->Args({4, 5, 6, 11}); + +BENCHMARK_MAIN(); From 01c0ab7cbb8afdf8af46baed06514f8fae7ee2fb Mon Sep 17 00:00:00 2001 From: Jeremy Ong Date: Thu, 27 Aug 2020 04:17:19 -0600 Subject: [PATCH 235/330] Fix Clang builds on Windows (#1021) Fixes #974. The `cxx_feature_check` now has an additional optional argument which can be used to supply extra cmake flags to pass to the `try_compile` command. The `CMAKE_CXX_STANDARD=14` flag was determined to be the minimum flag necessary to correctly compile and run the regex feature checks when compiling with Clang under Windows (n.b. this does *not* refer to clang-cl, the frontend to the MSVC compiler). The additional flag is not enabled for any other compiler/platform tuple. --- CMakeLists.txt | 13 ++++++++++--- cmake/CXXFeatureCheck.cmake | 5 +++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 67c0b70015..a157666148 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,11 +245,17 @@ if (BENCHMARK_USE_LIBCXX) endif() endif(BENCHMARK_USE_LIBCXX) +set(EXTRA_CXX_FLAGS "") +if (WIN32 AND "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + # Clang on Windows fails to compile the regex feature check under C++11 + set(EXTRA_CXX_FLAGS "-DCMAKE_CXX_STANDARD=14") +endif() + # C++ feature checks # Determine the correct regular expression engine to use -cxx_feature_check(STD_REGEX) -cxx_feature_check(GNU_POSIX_REGEX) -cxx_feature_check(POSIX_REGEX) +cxx_feature_check(STD_REGEX ${EXTRA_CXX_FLAGS}) +cxx_feature_check(GNU_POSIX_REGEX ${EXTRA_CXX_FLAGS}) +cxx_feature_check(POSIX_REGEX ${EXTRA_CXX_FLAGS}) if(NOT HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX) message(FATAL_ERROR "Failed to determine the source files for the regular expression backend") endif() @@ -257,6 +263,7 @@ if (NOT BENCHMARK_ENABLE_EXCEPTIONS AND HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX) message(WARNING "Using std::regex with exceptions disabled is not fully supported") endif() + cxx_feature_check(STEADY_CLOCK) # Ensure we have pthreads set(THREADS_PREFER_PTHREAD_FLAG ON) diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake index 059d510dd9..62e6741fe3 100644 --- a/cmake/CXXFeatureCheck.cmake +++ b/cmake/CXXFeatureCheck.cmake @@ -27,6 +27,11 @@ function(cxx_feature_check FILE) return() endif() + if (ARGC GREATER 1) + message(STATUS "Enabling additional flags: ${ARGV1}") + list(APPEND BENCHMARK_CXX_LINKER_FLAGS ${ARGV1}) + endif() + if (NOT DEFINED COMPILE_${FEATURE}) message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) From 4475ff6b8a7a4077d7492b76ef5278a3dc53a2e4 Mon Sep 17 00:00:00 2001 From: Yesudeep Mangalapilly Date: Fri, 28 Aug 2020 00:46:12 -0700 Subject: [PATCH 236/330] =?UTF-8?q?Adds=20-lm=20linker=20flag=20for=20Free?= =?UTF-8?q?BSD/OpenBSD=20and=20uses=20github.com/bazelbuil=E2=80=A6=20(#10?= =?UTF-8?q?32)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds -lm linker flag for (Free|Open)BSD and uses github.com/bazelbuild/platforms for platform detection. * Prefer selects.with_or to select the linkopts. * @platforms appears to be implicitly available. @bazel_skylib would require updating every dependent repository. * Re-enable platforms package. --- BUILD.bazel | 31 +++++++++++++++++++++++++++---- WORKSPACE | 5 +++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index d97a019bee..e38ebd72d3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,14 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + licenses(["notice"]) +config_setting( + name = "freebsd", + constraint_values = [ + "@platforms//os:freebsd", + ], + visibility = [":__subpackages__"], +) + +config_setting( + name = "openbsd", + constraint_values = [ + "@platforms//os:openbsd", + ], + visibility = [":__subpackages__"], +) + config_setting( name = "windows", - values = { - "cpu": "x64_windows", - }, + constraint_values = [ + "@platforms//os:windows", + ], visibility = [":__subpackages__"], ) -load("@rules_cc//cc:defs.bzl", "cc_library") +BSD_LINKOPTS = [ + "-pthread", + "-lm", +] cc_library( name = "benchmark", @@ -22,6 +43,8 @@ cc_library( hdrs = ["include/benchmark/benchmark.h"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], + ":freebsd": BSD_LINKOPTS, + ":openbsd": BSD_LINKOPTS, "//conditions:default": ["-pthread"], }), strip_include_prefix = "include", diff --git a/WORKSPACE b/WORKSPACE index 5438ad3611..7f2612b3bd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -35,3 +35,8 @@ new_local_repository( path = "/usr/include/python3.6", # May be overwritten by setup.py. ) +http_archive( + name = "platforms", + strip_prefix = "platforms-master", + urls = ["https://github.com/bazelbuild/platforms/archive/master.zip"], +) From 4751550871a4765c027d39680b842f590e1192b2 Mon Sep 17 00:00:00 2001 From: Yesudeep Mangalapilly Date: Thu, 3 Sep 2020 02:59:15 -0700 Subject: [PATCH 237/330] Revert previous linker additions for FreeBSD as the problem is Bazel using /usr/bin/clang instead of /usr/bin/clang++ to link C++ code. (#1035) --- BUILD.bazel | 29 +++-------------------------- WORKSPACE | 6 ------ 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index e38ebd72d3..eb35b62730 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,35 +2,14 @@ load("@rules_cc//cc:defs.bzl", "cc_library") licenses(["notice"]) -config_setting( - name = "freebsd", - constraint_values = [ - "@platforms//os:freebsd", - ], - visibility = [":__subpackages__"], -) - -config_setting( - name = "openbsd", - constraint_values = [ - "@platforms//os:openbsd", - ], - visibility = [":__subpackages__"], -) - config_setting( name = "windows", - constraint_values = [ - "@platforms//os:windows", - ], + values = { + "cpu": "x64_windows", + }, visibility = [":__subpackages__"], ) -BSD_LINKOPTS = [ - "-pthread", - "-lm", -] - cc_library( name = "benchmark", srcs = glob( @@ -43,8 +22,6 @@ cc_library( hdrs = ["include/benchmark/benchmark.h"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], - ":freebsd": BSD_LINKOPTS, - ":openbsd": BSD_LINKOPTS, "//conditions:default": ["-pthread"], }), strip_include_prefix = "include", diff --git a/WORKSPACE b/WORKSPACE index 7f2612b3bd..c00d12cd17 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -34,9 +34,3 @@ new_local_repository( build_file = "@//bindings/python:python_headers.BUILD", path = "/usr/include/python3.6", # May be overwritten by setup.py. ) - -http_archive( - name = "platforms", - strip_prefix = "platforms-master", - urls = ["https://github.com/bazelbuild/platforms/archive/master.zip"], -) From beb360d03e2a1a2673d9c2cf408c13b69fdb5627 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 9 Sep 2020 09:43:26 +0100 Subject: [PATCH 238/330] Create pylint.yml (#1039) * Create pylint.yml * improve file matching * fix some pylint issues * run on PR and push (force on master only) * more pylint fixes * suppress noisy exit code and filter to fatals * add conan as a dep so the module is importable * fix lint error on unreachable branch --- .github/workflows/build-and-test.yml | 8 +- .github/workflows/pylint.yml | 26 ++++ bindings/python/google_benchmark/__init__.py | 24 ++-- bindings/python/google_benchmark/example.py | 18 +-- setup.py | 139 ++++++++++--------- tools/gbench/util.py | 5 +- 6 files changed, 124 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 599b0943d0..f0f0626d74 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -18,20 +18,20 @@ jobs: build_type: ['Release', 'Debug'] steps: - uses: actions/checkout@v2 - + - name: create build environment run: cmake -E make_directory ${{ runner.workspace }}/_build - + - name: configure cmake shell: bash working-directory: ${{ runner.workspace }}/_build run: cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - + - name: build shell: bash working-directory: ${{ runner.workspace }}/_build run: cmake --build . --config ${{ matrix.build_type }} - + - name: test shell: bash working-directory: ${{ runner.workspace }}/_build diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000000..c8696749f3 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,26 @@ +name: pylint + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + pylint: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint pylint-exit conan + - name: Run pylint + run: | + pylint `find . -name '*.py'|xargs` || pylint-exit $? diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index c3a93bfc23..44531f9996 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -39,27 +39,27 @@ def my_benchmark(state): def register(f=None, *, name=None): - if f is None: - return lambda f: register(f, name=name) - if name is None: - name = f.__name__ - _benchmark.RegisterBenchmark(name, f) - return f + if f is None: + return lambda f: register(f, name=name) + if name is None: + name = f.__name__ + _benchmark.RegisterBenchmark(name, f) + return f def _flags_parser(argv): - argv = _benchmark.Initialize(argv) - return app.parse_flags_with_usage(argv) + argv = _benchmark.Initialize(argv) + return app.parse_flags_with_usage(argv) def _run_benchmarks(argv): - if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') - return _benchmark.RunSpecifiedBenchmarks() + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + return _benchmark.RunSpecifiedBenchmarks() def main(argv=None): - return app.run(_run_benchmarks, argv=argv, flags_parser=_flags_parser) + return app.run(_run_benchmarks, argv=argv, flags_parser=_flags_parser) # Methods for use with custom main function. diff --git a/bindings/python/google_benchmark/example.py b/bindings/python/google_benchmark/example.py index e968462479..0dead754c5 100644 --- a/bindings/python/google_benchmark/example.py +++ b/bindings/python/google_benchmark/example.py @@ -25,24 +25,24 @@ @benchmark.register def empty(state): - while state: - pass + while state: + pass @benchmark.register def sum_million(state): - while state: - sum(range(1_000_000)) + while state: + sum(range(1_000_000)) @benchmark.register def skipped(state): - if True: # Test some predicate here. - state.skip_with_error('some error') - return # NOTE: You must explicitly return, or benchmark will continue. + if True: # Test some predicate here. + state.skip_with_error('some error') + return # NOTE: You must explicitly return, or benchmark will continue. - ... # Benchmark code would be here. + # Benchmark code would be here. if __name__ == '__main__': - benchmark.main() + benchmark.main() diff --git a/setup.py b/setup.py index a2b0b917ad..800a87941b 100644 --- a/setup.py +++ b/setup.py @@ -9,89 +9,91 @@ from setuptools.command import build_ext -here = os.path.dirname(os.path.abspath(__file__)) +HERE = os.path.dirname(os.path.abspath(__file__)) IS_WINDOWS = sys.platform.startswith('win') def _get_version(): - """Parse the version string from __init__.py.""" - with open(os.path.join(here, 'bindings', 'python', 'google_benchmark', '__init__.py')) as f: - try: - version_line = next( - line for line in f if line.startswith('__version__')) - except StopIteration: - raise ValueError('__version__ not defined in __init__.py') - else: - ns = {} - exec(version_line, ns) # pylint: disable=exec-used - return ns['__version__'] + """Parse the version string from __init__.py.""" + with open(os.path.join( + HERE, 'bindings', 'python', 'google_benchmark', '__init__.py')) as init_file: + try: + version_line = next( + line for line in init_file if line.startswith('__version__')) + except StopIteration: + raise ValueError('__version__ not defined in __init__.py') + else: + namespace = {} + exec(version_line, namespace) # pylint: disable=exec-used + return namespace['__version__'] def _parse_requirements(path): - with open(os.path.join(here, path)) as f: - return [ - line.rstrip() for line in f - if not (line.isspace() or line.startswith('#')) - ] + with open(os.path.join(HERE, path)) as requirements: + return [ + line.rstrip() for line in requirements + if not (line.isspace() or line.startswith('#')) + ] class BazelExtension(setuptools.Extension): - """A C/C++ extension that is defined as a Bazel BUILD target.""" + """A C/C++ extension that is defined as a Bazel BUILD target.""" - def __init__(self, name, bazel_target): - self.bazel_target = bazel_target - self.relpath, self.target_name = ( - posixpath.relpath(bazel_target, '//').split(':')) - setuptools.Extension.__init__(self, name, sources=[]) + def __init__(self, name, bazel_target): + self.bazel_target = bazel_target + self.relpath, self.target_name = ( + posixpath.relpath(bazel_target, '//').split(':')) + setuptools.Extension.__init__(self, name, sources=[]) class BuildBazelExtension(build_ext.build_ext): - """A command that runs Bazel to build a C/C++ extension.""" - - def run(self): - for ext in self.extensions: - self.bazel_build(ext) - build_ext.build_ext.run(self) - - def bazel_build(self, ext): - with open('WORKSPACE', 'r') as f: - workspace_contents = f.read() - - with open('WORKSPACE', 'w') as f: - f.write(re.sub( - r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', - sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), - workspace_contents)) - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - - bazel_argv = [ - 'bazel', - 'build', - ext.bazel_target, - '--symlink_prefix=' + os.path.join(self.build_temp, 'bazel-'), - '--compilation_mode=' + ('dbg' if self.debug else 'opt'), - ] - - if IS_WINDOWS: - # Link with python*.lib. - for library_dir in self.library_dirs: - bazel_argv.append('--linkopt=/LIBPATH:' + library_dir) - - self.spawn(bazel_argv) - - shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' - ext_bazel_bin_path = os.path.join( - self.build_temp, 'bazel-bin', - ext.relpath, ext.target_name + shared_lib_suffix) - ext_dest_path = self.get_ext_fullpath(ext.name) - ext_dest_dir = os.path.dirname(ext_dest_path) - if not os.path.exists(ext_dest_dir): - os.makedirs(ext_dest_dir) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + """A command that runs Bazel to build a C/C++ extension.""" + + def run(self): + for ext in self.extensions: + self.bazel_build(ext) + build_ext.build_ext.run(self) + + def bazel_build(self, ext): + """Runs the bazel build to create the package.""" + with open('WORKSPACE', 'r') as workspace: + workspace_contents = workspace.read() + + with open('WORKSPACE', 'w') as workspace: + workspace.write(re.sub( + r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', + sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), + workspace_contents)) + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + bazel_argv = [ + 'bazel', + 'build', + ext.bazel_target, + '--symlink_prefix=' + os.path.join(self.build_temp, 'bazel-'), + '--compilation_mode=' + ('dbg' if self.debug else 'opt'), + ] + + if IS_WINDOWS: + # Link with python*.lib. + for library_dir in self.library_dirs: + bazel_argv.append('--linkopt=/LIBPATH:' + library_dir) + + self.spawn(bazel_argv) + + shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' + ext_bazel_bin_path = os.path.join( + self.build_temp, 'bazel-bin', + ext.relpath, ext.target_name + shared_lib_suffix) + ext_dest_path = self.get_ext_fullpath(ext.name) + ext_dest_dir = os.path.dirname(ext_dest_path) + if not os.path.exists(ext_dest_dir): + os.makedirs(ext_dest_dir) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) setuptools.setup( @@ -106,7 +108,8 @@ def bazel_build(self, ext): packages=setuptools.find_packages('bindings/python'), install_requires=_parse_requirements('bindings/python/requirements.txt'), cmdclass=dict(build_ext=BuildBazelExtension), - ext_modules=[BazelExtension('google_benchmark._benchmark', '//bindings/python/google_benchmark:_benchmark')], + ext_modules=[BazelExtension( + 'google_benchmark._benchmark', '//bindings/python/google_benchmark:_benchmark')], zip_safe=False, # PyPI package information. classifiers=[ diff --git a/tools/gbench/util.py b/tools/gbench/util.py index 1f8e8e2c47..661c4bad8d 100644 --- a/tools/gbench/util.py +++ b/tools/gbench/util.py @@ -158,7 +158,6 @@ def run_or_load_benchmark(filename, benchmark_flags): ftype = check_input_file(filename) if ftype == IT_JSON: return load_benchmark_results(filename) - elif ftype == IT_Executable: + if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) - else: - assert False # This branch is unreachable + raise ValueError('Unknown file type %s' % ftype) From 12e85b2eeb6df095b1366a44d8e8464599b5e1b8 Mon Sep 17 00:00:00 2001 From: Antoine Prouvost Date: Thu, 10 Sep 2020 04:57:30 -0400 Subject: [PATCH 239/330] Bind more State methods/attributes to Python (#1037) * Bind Counter to Python * Bind State methods to Python * Bind state.counters to Python * Import _benchmark.Counter * Add Python example of state usage Co-authored-by: Dominic Hamon --- bindings/python/google_benchmark/__init__.py | 4 +- bindings/python/google_benchmark/benchmark.cc | 74 +++++++++++++++++-- bindings/python/google_benchmark/example.py | 54 +++++++++++++- 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index 44531f9996..7ddbe2fb4a 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -29,10 +29,12 @@ def my_benchmark(state): from absl import app from google_benchmark import _benchmark +from google_benchmark._benchmark import Counter __all__ = [ "register", "main", + "Counter", ] __version__ = "0.1.0" @@ -54,7 +56,7 @@ def _flags_parser(argv): def _run_benchmarks(argv): if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') + raise app.UsageError("Too many command-line arguments.") return _benchmark.RunSpecifiedBenchmarks() diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index 374bf5479c..4e8515ff2c 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -1,8 +1,17 @@ // Benchmark for Python. #include "benchmark/benchmark.h" + +#include +#include +#include + +#include "pybind11/operators.h" #include "pybind11/pybind11.h" #include "pybind11/stl.h" +#include "pybind11/stl_bind.h" + +PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); namespace { namespace py = ::pybind11; @@ -29,9 +38,8 @@ std::vector Initialize(const std::vector& argv) { } void RegisterBenchmark(const char* name, py::function f) { - benchmark::RegisterBenchmark(name, [f](benchmark::State& state) { - f(&state); - }); + benchmark::RegisterBenchmark(name, + [f](benchmark::State& state) { f(&state); }); } PYBIND11_MODULE(_benchmark, m) { @@ -40,9 +48,61 @@ PYBIND11_MODULE(_benchmark, m) { m.def("RunSpecifiedBenchmarks", []() { benchmark::RunSpecifiedBenchmarks(); }); - py::class_(m, "State") - .def("__bool__", &benchmark::State::KeepRunning) - .def_property_readonly("keep_running", &benchmark::State::KeepRunning) - .def("skip_with_error", &benchmark::State::SkipWithError); + using benchmark::Counter; + py::class_ py_counter(m, "Counter"); + + py::enum_(py_counter, "Flags") + .value("kDefaults", Counter::Flags::kDefaults) + .value("kIsRate", Counter::Flags::kIsRate) + .value("kAvgThreads", Counter::Flags::kAvgThreads) + .value("kAvgThreadsRate", Counter::Flags::kAvgThreadsRate) + .value("kIsIterationInvariant", Counter::Flags::kIsIterationInvariant) + .value("kIsIterationInvariantRate", + Counter::Flags::kIsIterationInvariantRate) + .value("kAvgIterations", Counter::Flags::kAvgIterations) + .value("kAvgIterationsRate", Counter::Flags::kAvgIterationsRate) + .value("kInvert", Counter::Flags::kInvert) + .export_values() + .def(py::self | py::self); + + py::enum_(py_counter, "OneK") + .value("kIs1000", Counter::OneK::kIs1000) + .value("kIs1024", Counter::OneK::kIs1024) + .export_values(); + + py_counter + .def(py::init(), + py::arg("value") = 0., py::arg("flags") = Counter::kDefaults, + py::arg("k") = Counter::kIs1000) + .def(py::init([](double value) { return Counter(value); })) + .def_readwrite("value", &Counter::value) + .def_readwrite("flags", &Counter::flags) + .def_readwrite("oneK", &Counter::oneK); + py::implicitly_convertible(); + py::implicitly_convertible(); + + py::bind_map(m, "UserCounters"); + + using benchmark::State; + py::class_(m, "State") + .def("__bool__", &State::KeepRunning) + .def_property_readonly("keep_running", &State::KeepRunning) + .def("pause_timing", &State::PauseTiming) + .def("resume_timing", &State::ResumeTiming) + .def("skip_with_error", &State::SkipWithError) + .def_property_readonly("error_occured", &State::error_occurred) + .def("set_iteration_time", &State::SetIterationTime) + .def_property("bytes_processed", &State::bytes_processed, + &State::SetBytesProcessed) + .def_property("complexity_n", &State::complexity_length_n, + &State::SetComplexityN) + .def_property("items_processed", &State::items_processed, + &State::SetItemsProcessed) + .def("set_label", (void (State::*)(const char*)) & State::SetLabel) + .def("range", &State::range, py::arg("pos") = 0) + .def_property_readonly("iterations", &State::iterations) + .def_readwrite("counters", &State::counters) + .def_readonly("thread_index", &State::thread_index) + .def_readonly("threads", &State::threads); }; } // namespace diff --git a/bindings/python/google_benchmark/example.py b/bindings/python/google_benchmark/example.py index 0dead754c5..9bb23c450e 100644 --- a/bindings/python/google_benchmark/example.py +++ b/bindings/python/google_benchmark/example.py @@ -20,7 +20,11 @@ python setup.py install """ +import random +import time + import google_benchmark as benchmark +from google_benchmark import Counter @benchmark.register @@ -34,15 +38,59 @@ def sum_million(state): while state: sum(range(1_000_000)) +@benchmark.register +def pause_timing(state): + """Pause timing every iteration.""" + while state: + # Construct a list of random ints every iteration without timing it + state.pause_timing() + random_list = [random.randint(0, 100) for _ in range(100)] + state.resume_timing() + # Time the in place sorting algorithm + random_list.sort() + @benchmark.register def skipped(state): if True: # Test some predicate here. - state.skip_with_error('some error') + state.skip_with_error("some error") return # NOTE: You must explicitly return, or benchmark will continue. - # Benchmark code would be here. + ... # Benchmark code would be here. + + +@benchmark.register +def manual_timing(state): + while state: + # Manually count Python CPU time + start = time.perf_counter() # perf_counter_ns() in Python 3.7+ + # Somehting to benchmark + time.sleep(0.01) + end = time.perf_counter() + state.set_iteration_time(end - start) + + +@benchmark.register +def custom_counters(state): + """Collect cutom metric using benchmark.Counter.""" + num_foo = 0.0 + while state: + # Benchmark some code here + pass + # Collect some custom metric named foo + num_foo += 0.13 + + # Automatic Counter from numbers. + state.counters["foo"] = num_foo + # Set a counter as a rate. + state.counters["foo_rate"] = Counter(num_foo, Counter.kIsRate) + # Set a counter as an inverse of rate. + state.counters["foo_inv_rate"] = Counter(num_foo, Counter.kIsRate | Counter.kInvert) + # Set a counter as a thread-average quantity. + state.counters["foo_avg"] = Counter(num_foo, Counter.kAvgThreads) + # There's also a combined flag: + state.counters["foo_avg_rate"] = Counter(num_foo, Counter.kAvgThreadsRate) -if __name__ == '__main__': +if __name__ == "__main__": benchmark.main() From df9e2948fa7bfca1ddf530ae2b23a518ed55fab1 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 10 Sep 2020 16:32:25 +0100 Subject: [PATCH 240/330] Add workflow to exercise bindings (#1041) * Initial version to try to run python bindings example * python indent issue in setup.py * better naming --- .github/workflows/test_bindings.yml | 24 ++++++++++++++++++++++++ setup.py | 22 +++++++++++----------- 2 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/test_bindings.yml diff --git a/.github/workflows/test_bindings.yml b/.github/workflows/test_bindings.yml new file mode 100644 index 0000000000..273d7f93ee --- /dev/null +++ b/.github/workflows/test_bindings.yml @@ -0,0 +1,24 @@ +name: test-bindings + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + python_bindings: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install benchmark + run: + python setup.py install + - name: Run example bindings + run: + python bindings/python/google_benchmark/example.py diff --git a/setup.py b/setup.py index 800a87941b..a63795c99a 100644 --- a/setup.py +++ b/setup.py @@ -83,17 +83,17 @@ def bazel_build(self, ext): for library_dir in self.library_dirs: bazel_argv.append('--linkopt=/LIBPATH:' + library_dir) - self.spawn(bazel_argv) - - shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' - ext_bazel_bin_path = os.path.join( - self.build_temp, 'bazel-bin', - ext.relpath, ext.target_name + shared_lib_suffix) - ext_dest_path = self.get_ext_fullpath(ext.name) - ext_dest_dir = os.path.dirname(ext_dest_path) - if not os.path.exists(ext_dest_dir): - os.makedirs(ext_dest_dir) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + self.spawn(bazel_argv) + + shared_lib_suffix = '.dll' if IS_WINDOWS else '.so' + ext_bazel_bin_path = os.path.join( + self.build_temp, 'bazel-bin', + ext.relpath, ext.target_name + shared_lib_suffix) + ext_dest_path = self.get_ext_fullpath(ext.name) + ext_dest_dir = os.path.dirname(ext_dest_path) + if not os.path.exists(ext_dest_dir): + os.makedirs(ext_dest_dir) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) setuptools.setup( From 73d4d5e8d6d449fc8663765a42aa8aeeee844489 Mon Sep 17 00:00:00 2001 From: Antoine Prouvost Date: Fri, 11 Sep 2020 05:55:18 -0400 Subject: [PATCH 241/330] Bind benchmark builder to Python (#1040) * Fix setup.py and reformat * Bind benchmark * Add benchmark option to Python * Add Python examples for range, complexity, and thread * Remove invalid multithreading in Python * Bump Python bindings version to 0.2.0 Co-authored-by: Dominic Hamon --- bindings/python/google_benchmark/__init__.py | 101 ++++++++++++++++-- bindings/python/google_benchmark/benchmark.cc | 90 ++++++++++++++-- bindings/python/google_benchmark/example.py | 42 +++++++- setup.py | 96 +++++++++-------- 4 files changed, 270 insertions(+), 59 deletions(-) diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index 7ddbe2fb4a..787c423d5d 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -29,24 +29,111 @@ def my_benchmark(state): from absl import app from google_benchmark import _benchmark -from google_benchmark._benchmark import Counter +from google_benchmark._benchmark import ( + Counter, + kNanosecond, + kMicrosecond, + kMillisecond, + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto, + oLambda, +) + __all__ = [ "register", "main", "Counter", + "kNanosecond", + "kMicrosecond", + "kMillisecond", + "oNone", + "o1", + "oN", + "oNSquared", + "oNCubed", + "oLogN", + "oNLogN", + "oAuto", + "oLambda", ] -__version__ = "0.1.0" +__version__ = "0.2.0" + + +class __OptionMaker: + """A stateless class to collect benchmark options. + + Collect all decorator calls like @option.range(start=0, limit=1<<5). + """ + + class Options: + """Pure data class to store options calls, along with the benchmarked function.""" + + def __init__(self, func): + self.func = func + self.builder_calls = [] + + @classmethod + def make(cls, func_or_options): + """Make Options from Options or the benchmarked function.""" + if isinstance(func_or_options, cls.Options): + return func_or_options + return cls.Options(func_or_options) + + def __getattr__(self, builder_name): + """Append option call in the Options.""" + + # The function that get returned on @option.range(start=0, limit=1<<5). + def __builder_method(*args, **kwargs): + # The decorator that get called, either with the benchmared function + # or the previous Options + def __decorator(func_or_options): + options = self.make(func_or_options) + options.builder_calls.append((builder_name, args, kwargs)) + # The decorator returns Options so it is not technically a decorator + # and needs a final call to @regiser + return options -def register(f=None, *, name=None): - if f is None: + return __decorator + + return __builder_method + + +# Alias for nicer API. +# We have to instanciate an object, even if stateless, to be able to use __getattr__ +# on option.range +option = __OptionMaker() + + +def register(undefined=None, *, name=None): + """Register function for benchmarking.""" + if undefined is None: + # Decorator is called without parenthesis so we return a decorator return lambda f: register(f, name=name) + + # We have either the function to benchmark (simple case) or an instance of Options + # (@option._ case). + options = __OptionMaker.make(undefined) + if name is None: - name = f.__name__ - _benchmark.RegisterBenchmark(name, f) - return f + name = options.func.__name__ + + # We register the benchmark and reproduce all the @option._ calls onto the + # benchmark builder pattern + benchmark = _benchmark.RegisterBenchmark(name, options.func) + for name, args, kwargs in options.builder_calls[::-1]: + getattr(benchmark, name)(*args, **kwargs) + + # return the benchmarked function because the decorator does not modify it + return options.func def _flags_parser(argv): diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index 4e8515ff2c..a733339769 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -1,7 +1,5 @@ // Benchmark for Python. -#include "benchmark/benchmark.h" - #include #include #include @@ -11,6 +9,8 @@ #include "pybind11/stl.h" #include "pybind11/stl_bind.h" +#include "benchmark/benchmark.h" + PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); namespace { @@ -37,16 +37,82 @@ std::vector Initialize(const std::vector& argv) { return remaining_argv; } -void RegisterBenchmark(const char* name, py::function f) { - benchmark::RegisterBenchmark(name, - [f](benchmark::State& state) { f(&state); }); +benchmark::internal::Benchmark* RegisterBenchmark(const char* name, + py::function f) { + return benchmark::RegisterBenchmark( + name, [f](benchmark::State& state) { f(&state); }); } PYBIND11_MODULE(_benchmark, m) { - m.def("Initialize", Initialize); - m.def("RegisterBenchmark", RegisterBenchmark); - m.def("RunSpecifiedBenchmarks", - []() { benchmark::RunSpecifiedBenchmarks(); }); + using benchmark::TimeUnit; + py::enum_(m, "TimeUnit") + .value("kNanosecond", TimeUnit::kNanosecond) + .value("kMicrosecond", TimeUnit::kMicrosecond) + .value("kMillisecond", TimeUnit::kMillisecond) + .export_values(); + + using benchmark::BigO; + py::enum_(m, "BigO") + .value("oNone", BigO::oNone) + .value("o1", BigO::o1) + .value("oN", BigO::oN) + .value("oNSquared", BigO::oNSquared) + .value("oNCubed", BigO::oNCubed) + .value("oLogN", BigO::oLogN) + .value("oNLogN", BigO::oLogN) + .value("oAuto", BigO::oAuto) + .value("oLambda", BigO::oLambda) + .export_values(); + + using benchmark::internal::Benchmark; + py::class_(m, "Benchmark") + // For methods returning a pointer tor the current object, reference + // return policy is used to ask pybind not to take ownership oof the + // returned object and avoid calling delete on it. + // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies + // + // For methods taking a const std::vector<...>&, a copy is created + // because a it is bound to a Python list. + // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html + .def("unit", &Benchmark::Unit, py::return_value_policy::reference) + .def("arg", &Benchmark::Arg, py::return_value_policy::reference) + .def("args", &Benchmark::Args, py::return_value_policy::reference) + .def("range", &Benchmark::Range, py::return_value_policy::reference, + py::arg("start"), py::arg("limit")) + .def("dense_range", &Benchmark::DenseRange, + py::return_value_policy::reference, py::arg("start"), + py::arg("limit"), py::arg("step") = 1) + .def("ranges", &Benchmark::Ranges, py::return_value_policy::reference) + .def("args_product", &Benchmark::ArgsProduct, + py::return_value_policy::reference) + .def("arg_name", &Benchmark::ArgName, py::return_value_policy::reference) + .def("arg_names", &Benchmark::ArgNames, + py::return_value_policy::reference) + .def("range_pair", &Benchmark::RangePair, + py::return_value_policy::reference, py::arg("lo1"), py::arg("hi1"), + py::arg("lo2"), py::arg("hi2")) + .def("range_multiplier", &Benchmark::RangeMultiplier, + py::return_value_policy::reference) + .def("min_time", &Benchmark::MinTime, py::return_value_policy::reference) + .def("iterations", &Benchmark::Iterations, + py::return_value_policy::reference) + .def("repetitions", &Benchmark::Repetitions, + py::return_value_policy::reference) + .def("report_aggregates_only", &Benchmark::ReportAggregatesOnly, + py::return_value_policy::reference, py::arg("value") = true) + .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, + py::return_value_policy::reference, py::arg("value") = true) + .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, + py::return_value_policy::reference) + .def("use_real_time", &Benchmark::UseRealTime, + py::return_value_policy::reference) + .def("use_manual_time", &Benchmark::UseManualTime, + py::return_value_policy::reference) + .def( + "complexity", + (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity, + py::return_value_policy::reference, + py::arg("complexity") = benchmark::oAuto); using benchmark::Counter; py::class_ py_counter(m, "Counter"); @@ -104,5 +170,11 @@ PYBIND11_MODULE(_benchmark, m) { .def_readwrite("counters", &State::counters) .def_readonly("thread_index", &State::thread_index) .def_readonly("threads", &State::threads); + + m.def("Initialize", Initialize); + m.def("RegisterBenchmark", RegisterBenchmark, + py::return_value_policy::reference); + m.def("RunSpecifiedBenchmarks", + []() { benchmark::RunSpecifiedBenchmarks(); }); }; } // namespace diff --git a/bindings/python/google_benchmark/example.py b/bindings/python/google_benchmark/example.py index 9bb23c450e..9134e8cffe 100644 --- a/bindings/python/google_benchmark/example.py +++ b/bindings/python/google_benchmark/example.py @@ -64,7 +64,7 @@ def manual_timing(state): while state: # Manually count Python CPU time start = time.perf_counter() # perf_counter_ns() in Python 3.7+ - # Somehting to benchmark + # Something to benchmark time.sleep(0.01) end = time.perf_counter() state.set_iteration_time(end - start) @@ -92,5 +92,45 @@ def custom_counters(state): state.counters["foo_avg_rate"] = Counter(num_foo, Counter.kAvgThreadsRate) +@benchmark.register +@benchmark.option.measure_process_cpu_time() +@benchmark.option.use_real_time() +def with_options(state): + while state: + sum(range(1_000_000)) + + +@benchmark.register(name="sum_million_microseconds") +@benchmark.option.unit(benchmark.kMicrosecond) +def with_options(state): + while state: + sum(range(1_000_000)) + + +@benchmark.register +@benchmark.option.arg(100) +@benchmark.option.arg(1000) +def passing_argument(state): + while state: + sum(range(state.range(0))) + + +@benchmark.register +@benchmark.option.range(8, limit=8 << 10) +def using_range(state): + while state: + sum(range(state.range(0))) + + +@benchmark.register +@benchmark.option.range_multiplier(2) +@benchmark.option.range(1 << 10, 1 << 18) +@benchmark.option.complexity(benchmark.oN) +def computing_complexity(state): + while state: + sum(range(state.range(0))) + state.complexity_n = state.range(0) + + if __name__ == "__main__": benchmark.main() diff --git a/setup.py b/setup.py index a63795c99a..5cdab10cf7 100644 --- a/setup.py +++ b/setup.py @@ -12,29 +12,32 @@ HERE = os.path.dirname(os.path.abspath(__file__)) -IS_WINDOWS = sys.platform.startswith('win') +IS_WINDOWS = sys.platform.startswith("win") def _get_version(): """Parse the version string from __init__.py.""" - with open(os.path.join( - HERE, 'bindings', 'python', 'google_benchmark', '__init__.py')) as init_file: + with open( + os.path.join(HERE, "bindings", "python", "google_benchmark", "__init__.py") + ) as init_file: try: version_line = next( - line for line in init_file if line.startswith('__version__')) + line for line in init_file if line.startswith("__version__") + ) except StopIteration: - raise ValueError('__version__ not defined in __init__.py') + raise ValueError("__version__ not defined in __init__.py") else: namespace = {} exec(version_line, namespace) # pylint: disable=exec-used - return namespace['__version__'] + return namespace["__version__"] def _parse_requirements(path): with open(os.path.join(HERE, path)) as requirements: return [ - line.rstrip() for line in requirements - if not (line.isspace() or line.startswith('#')) + line.rstrip() + for line in requirements + if not (line.isspace() or line.startswith("#")) ] @@ -43,8 +46,9 @@ class BazelExtension(setuptools.Extension): def __init__(self, name, bazel_target): self.bazel_target = bazel_target - self.relpath, self.target_name = ( - posixpath.relpath(bazel_target, '//').split(':')) + self.relpath, self.target_name = posixpath.relpath(bazel_target, "//").split( + ":" + ) setuptools.Extension.__init__(self, name, sources=[]) @@ -58,30 +62,33 @@ def run(self): def bazel_build(self, ext): """Runs the bazel build to create the package.""" - with open('WORKSPACE', 'r') as workspace: + with open("WORKSPACE", "r") as workspace: workspace_contents = workspace.read() - with open('WORKSPACE', 'w') as workspace: - workspace.write(re.sub( - r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', - sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), - workspace_contents)) + with open("WORKSPACE", "w") as workspace: + workspace.write( + re.sub( + r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', + sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), + workspace_contents, + ) + ) if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) bazel_argv = [ - 'bazel', - 'build', + "bazel", + "build", ext.bazel_target, - '--symlink_prefix=' + os.path.join(self.build_temp, 'bazel-'), - '--compilation_mode=' + ('dbg' if self.debug else 'opt'), + "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), + "--compilation_mode=" + ("dbg" if self.debug else "opt"), ] if IS_WINDOWS: # Link with python*.lib. for library_dir in self.library_dirs: - bazel_argv.append('--linkopt=/LIBPATH:' + library_dir) + bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) self.spawn(bazel_argv) @@ -89,6 +96,7 @@ def bazel_build(self, ext): ext_bazel_bin_path = os.path.join( self.build_temp, 'bazel-bin', ext.relpath, ext.target_name + shared_lib_suffix) + ext_dest_path = self.get_ext_fullpath(ext.name) ext_dest_dir = os.path.dirname(ext_dest_path) if not os.path.exists(ext_dest_dir): @@ -97,32 +105,36 @@ def bazel_build(self, ext): setuptools.setup( - name='google_benchmark', + name="google_benchmark", version=_get_version(), - url='https://github.com/google/benchmark', - description='A library to benchmark code snippets.', - author='Google', - author_email='benchmark-py@google.com', + url="https://github.com/google/benchmark", + description="A library to benchmark code snippets.", + author="Google", + author_email="benchmark-py@google.com", # Contained modules and scripts. - package_dir={'': 'bindings/python'}, - packages=setuptools.find_packages('bindings/python'), - install_requires=_parse_requirements('bindings/python/requirements.txt'), + package_dir={"": "bindings/python"}, + packages=setuptools.find_packages("bindings/python"), + install_requires=_parse_requirements("bindings/python/requirements.txt"), cmdclass=dict(build_ext=BuildBazelExtension), - ext_modules=[BazelExtension( - 'google_benchmark._benchmark', '//bindings/python/google_benchmark:_benchmark')], + ext_modules=[ + BazelExtension( + "google_benchmark._benchmark", + "//bindings/python/google_benchmark:_benchmark", + ) + ], zip_safe=False, # PyPI package information. classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Software Development :: Testing', - 'Topic :: System :: Benchmark', + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Software Development :: Testing", + "Topic :: System :: Benchmark", ], - license='Apache 2.0', - keywords='benchmark', + license="Apache 2.0", + keywords="benchmark", ) From 7efada2dac40e9f6e5c5f76e0c4a78c85c0a3af5 Mon Sep 17 00:00:00 2001 From: Vitaly Zaitsev Date: Sat, 12 Sep 2020 17:56:43 +0200 Subject: [PATCH 242/330] Fixed pkg-config on other than Ubuntu GNU/Linux distributions. (#1043) Signed-off-by: Vitaly Zaitsev --- cmake/benchmark.pc.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/benchmark.pc.in b/cmake/benchmark.pc.in index 43ca8f91d7..34beb012ee 100644 --- a/cmake/benchmark.pc.in +++ b/cmake/benchmark.pc.in @@ -1,7 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/lib -includedir=${prefix}/include +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: @PROJECT_NAME@ Description: Google microbenchmark framework From 949f5bb11fb36ac0108c8e1cd238f738a24265f4 Mon Sep 17 00:00:00 2001 From: Jusufadis Bakamovic Date: Mon, 21 Sep 2020 12:25:28 +0200 Subject: [PATCH 243/330] Add support for JSON dumps of benchmark diff reports. (#1042). Fixes #737. NOTE: This is a fresh-start of #738 pull-request which I messed up by re-editing the commiter email which I forgot to modify before pushing. Sorry for the inconvenience. This PR brings proposed solution for functionality described in #737 Fixes #737. --- tools/compare.py | 19 +- tools/gbench/report.py | 620 ++++++++++++++++++++++++++++++++--------- 2 files changed, 507 insertions(+), 132 deletions(-) diff --git a/tools/compare.py b/tools/compare.py index bd01be57cd..66eed932c2 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -7,6 +7,7 @@ import argparse from argparse import ArgumentParser +import json import sys import gbench from gbench import util, report @@ -56,6 +57,12 @@ def create_parser(): help="Do not use colors in the terminal output" ) + parser.add_argument( + '-d', + '--dump_to_json', + dest='dump_to_json', + help="Additionally, dump benchmark comparison output to this file in JSON format.") + utest = parser.add_argument_group() utest.add_argument( '--no-utest', @@ -244,14 +251,20 @@ def main(): json2 = gbench.report.filter_benchmark( json2_orig, filter_contender, replacement) - # Diff and output - output_lines = gbench.report.generate_difference_report( - json1, json2, args.display_aggregates_only, + diff_report = gbench.report.get_difference_report( + json1, json2, args.utest) + output_lines = gbench.report.print_difference_report( + diff_report, + args.display_aggregates_only, args.utest, args.utest_alpha, args.color) print(description) for ln in output_lines: print(ln) + # Optionally, diff and output to JSON + if args.dump_to_json is not None: + with open(args.dump_to_json, 'w') as f_json: + json.dump(diff_report, f_json) class TestParser(unittest.TestCase): def setUp(self): diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 5bd3a8d85d..bf29492ed9 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -154,6 +154,7 @@ def extract_field(partition, field_name): rhs = [x[field_name] for x in partition[1]] return [lhs, rhs] + def calc_utest(timings_cpu, timings_time): min_rep_cnt = min(len(timings_time[0]), len(timings_time[1]), @@ -171,46 +172,106 @@ def calc_utest(timings_cpu, timings_time): return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue -def print_utest(partition, utest_alpha, first_col_width, use_color=True): +def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True): def get_utest_color(pval): return BC_FAIL if pval >= utest_alpha else BC_OKGREEN - timings_time = extract_field(partition, 'real_time') - timings_cpu = extract_field(partition, 'cpu_time') - have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time) - # Check if we failed miserably with minimum required repetitions for utest - if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None: + if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None: return [] dsc = "U Test, Repetitions: {} vs {}".format( - len(timings_cpu[0]), len(timings_cpu[1])) + utest['nr_of_repetitions'], utest['nr_of_repetitions_other']) dsc_color = BC_OKGREEN # We still got some results to show but issue a warning about it. - if not have_optimal_repetitions: + if not utest['have_optimal_repetitions']: dsc_color = BC_WARNING dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format( UTEST_OPTIMAL_REPETITIONS) special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}" - last_name = partition[0][0]['name'] return [color_format(use_color, special_str, BC_HEADER, - "{}{}".format(last_name, UTEST_COL_NAME), + "{}{}".format(bc_name, UTEST_COL_NAME), first_col_width, - get_utest_color(time_pvalue), time_pvalue, - get_utest_color(cpu_pvalue), cpu_pvalue, + get_utest_color( + utest['time_pvalue']), utest['time_pvalue'], + get_utest_color( + utest['cpu_pvalue']), utest['cpu_pvalue'], dsc_color, dsc, endc=BC_ENDC)] -def generate_difference_report( +def get_difference_report( json1, json2, - display_aggregates_only=False, + utest=False): + """ + Calculate and report the difference between each test of two benchmarks + runs specified as 'json1' and 'json2'. Output is another json containing + relevant details for each test run. + """ + assert utest is True or utest is False + + diff_report = [] + partitions = partition_benchmarks(json1, json2) + for partition in partitions: + benchmark_name = partition[0][0]['name'] + time_unit = partition[0][0]['time_unit'] + measurements = [] + utest_results = {} + # Careful, we may have different repetition count. + for i in range(min(len(partition[0]), len(partition[1]))): + bn = partition[0][i] + other_bench = partition[1][i] + measurements.append({ + 'real_time': bn['real_time'], + 'cpu_time': bn['cpu_time'], + 'real_time_other': other_bench['real_time'], + 'cpu_time_other': other_bench['cpu_time'], + 'time': calculate_change(bn['real_time'], other_bench['real_time']), + 'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time']) + }) + + # After processing the whole partition, if requested, do the U test. + if utest: + timings_cpu = extract_field(partition, 'cpu_time') + timings_time = extract_field(partition, 'real_time') + have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time) + if cpu_pvalue and time_pvalue: + utest_results = { + 'have_optimal_repetitions': have_optimal_repetitions, + 'cpu_pvalue': cpu_pvalue, + 'time_pvalue': time_pvalue, + 'nr_of_repetitions': len(timings_cpu[0]), + 'nr_of_repetitions_other': len(timings_cpu[1]) + } + + # Store only if we had any measurements for given benchmark. + # E.g. partition_benchmarks will filter out the benchmarks having + # time units which are not compatible with other time units in the + # benchmark suite. + if measurements: + run_type = partition[0][0]['run_type'] if 'run_type' in partition[0][0] else '' + aggregate_name = partition[0][0]['aggregate_name'] if run_type == 'aggregate' and 'aggregate_name' in partition[0][0] else '' + diff_report.append({ + 'name': benchmark_name, + 'measurements': measurements, + 'time_unit': time_unit, + 'run_type': run_type, + 'aggregate_name': aggregate_name, + 'utest': utest_results + }) + + return diff_report + + +def print_difference_report( + json_diff_report, + include_aggregates_only=False, utest=False, utest_alpha=0.05, use_color=True): @@ -219,14 +280,16 @@ def generate_difference_report( runs specified as 'json1' and 'json2'. """ assert utest is True or utest is False - first_col_width = find_longest_name(json1['benchmarks']) - def find_test(name): - for b in json2['benchmarks']: - if b['name'] == name: - return b - return None + def get_color(res): + if res > 0.05: + return BC_FAIL + elif res > -0.07: + return BC_WHITE + else: + return BC_CYAN + first_col_width = find_longest_name(json_diff_report) first_col_width = max( first_col_width, len('Benchmark')) @@ -235,50 +298,36 @@ def find_test(name): 'Benchmark', 12 + first_col_width) output_strs = [first_line, '-' * len(first_line)] - partitions = partition_benchmarks(json1, json2) - for partition in partitions: - # Careful, we may have different repetition count. - for i in range(min(len(partition[0]), len(partition[1]))): - bn = partition[0][i] - other_bench = partition[1][i] + fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" + for benchmark in json_diff_report: + # *If* we were asked to only include aggregates, + # and if it is non-aggregate, then skip it. + if include_aggregates_only and 'run_type' in benchmark: + if benchmark['run_type'] != 'aggregate': + continue - # *If* we were asked to only display aggregates, - # and if it is non-aggregate, then skip it. - if display_aggregates_only and 'run_type' in bn and 'run_type' in other_bench: - assert bn['run_type'] == other_bench['run_type'] - if bn['run_type'] != 'aggregate': - continue - - fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" - - def get_color(res): - if res > 0.05: - return BC_FAIL - elif res > -0.07: - return BC_WHITE - else: - return BC_CYAN - - tres = calculate_change(bn['real_time'], other_bench['real_time']) - cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time']) + for measurement in benchmark['measurements']: output_strs += [color_format(use_color, fmt_str, BC_HEADER, - bn['name'], + benchmark['name'], first_col_width, - get_color(tres), - tres, - get_color(cpures), - cpures, - bn['real_time'], - other_bench['real_time'], - bn['cpu_time'], - other_bench['cpu_time'], + get_color(measurement['time']), + measurement['time'], + get_color(measurement['cpu']), + measurement['cpu'], + measurement['real_time'], + measurement['real_time_other'], + measurement['cpu_time'], + measurement['cpu_time_other'], endc=BC_ENDC)] - # After processing the whole partition, if requested, do the U test. - if utest: - output_strs += print_utest(partition, + # After processing the measurements, if requested and + # if applicable (e.g. u-test exists for given benchmark), + # print the U test. + if utest and benchmark['utest']: + output_strs += print_utest(benchmark['name'], + benchmark['utest'], utest_alpha=utest_alpha, first_col_width=first_col_width, use_color=use_color) @@ -319,21 +368,26 @@ def test_basic(self): class TestReportDifference(unittest.TestCase): - def load_results(self): - import json - testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test1_run1.json') - testOutput2 = os.path.join(testInputs, 'test1_run2.json') - with open(testOutput1, 'r') as f: - json1 = json.load(f) - with open(testOutput2, 'r') as f: - json2 = json.load(f) - return json1, json2 - - def test_basic(self): + @classmethod + def setUpClass(cls): + def load_results(): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput1 = os.path.join(testInputs, 'test1_run1.json') + testOutput2 = os.path.join(testInputs, 'test1_run2.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report(json1, json2) + + def test_json_diff_report_pretty_printing(self): expect_lines = [ ['BM_SameTimes', '+0.0000', '+0.0000', '10', '10', '10', '10'], ['BM_2xFaster', '-0.5000', '-0.5000', '50', '25', '50', '25'], @@ -351,9 +405,8 @@ def test_basic(self): ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'], ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'], ] - json1, json2 = self.load_results() - output_lines_with_header = generate_difference_report( - json1, json2, use_color=False) + output_lines_with_header = print_difference_report( + self.json_diff_report, use_color=False) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) @@ -363,31 +416,118 @@ def test_basic(self): self.assertEqual(len(parts), 7) self.assertEqual(expect_lines[i], parts) + def test_json_diff_report_output(self): + expected_output = [ + { + 'name': 'BM_SameTimes', + 'measurements': [{'time': 0.0000, 'cpu': 0.0000, 'real_time': 10, 'real_time_other': 10, 'cpu_time': 10, 'cpu_time_other': 10}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_2xFaster', + 'measurements': [{'time': -0.5000, 'cpu': -0.5000, 'real_time': 50, 'real_time_other': 25, 'cpu_time': 50, 'cpu_time_other': 25}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_2xSlower', + 'measurements': [{'time': 1.0000, 'cpu': 1.0000, 'real_time': 50, 'real_time_other': 100, 'cpu_time': 50, 'cpu_time_other': 100}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_1PercentFaster', + 'measurements': [{'time': -0.0100, 'cpu': -0.0100, 'real_time': 100, 'real_time_other': 98.9999999, 'cpu_time': 100, 'cpu_time_other': 98.9999999}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_1PercentSlower', + 'measurements': [{'time': 0.0100, 'cpu': 0.0100, 'real_time': 100, 'real_time_other': 101, 'cpu_time': 100, 'cpu_time_other': 101}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_10PercentFaster', + 'measurements': [{'time': -0.1000, 'cpu': -0.1000, 'real_time': 100, 'real_time_other': 90, 'cpu_time': 100, 'cpu_time_other': 90}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_10PercentSlower', + 'measurements': [{'time': 0.1000, 'cpu': 0.1000, 'real_time': 100, 'real_time_other': 110, 'cpu_time': 100, 'cpu_time_other': 110}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_100xSlower', + 'measurements': [{'time': 99.0000, 'cpu': 99.0000, 'real_time': 100, 'real_time_other': 10000, 'cpu_time': 100, 'cpu_time_other': 10000}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_100xFaster', + 'measurements': [{'time': -0.9900, 'cpu': -0.9900, 'real_time': 10000, 'real_time_other': 100, 'cpu_time': 10000, 'cpu_time_other': 100}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_10PercentCPUToTime', + 'measurements': [{'time': 0.1000, 'cpu': -0.1000, 'real_time': 100, 'real_time_other': 110, 'cpu_time': 100, 'cpu_time_other': 90}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_ThirdFaster', + 'measurements': [{'time': -0.3333, 'cpu': -0.3334, 'real_time': 100, 'real_time_other': 67, 'cpu_time': 100, 'cpu_time_other': 67}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': 'BM_NotBadTimeUnit', + 'measurements': [{'time': -0.9000, 'cpu': 0.2000, 'real_time': 0.4, 'real_time_other': 0.04, 'cpu_time': 0.5, 'cpu_time_other': 0.6}], + 'time_unit': 's', + 'utest': {} + }, + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip( + self.json_diff_report, expected_output): + self.assertEqual(out['name'], expected['name']) + self.assertEqual(out['time_unit'], expected['time_unit']) + assert_utest(self, out, expected) + assert_measurements(self, out, expected) + class TestReportDifferenceBetweenFamilies(unittest.TestCase): - def load_result(self): - import json - testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput = os.path.join(testInputs, 'test2_run.json') - with open(testOutput, 'r') as f: - json = json.load(f) - return json + @classmethod + def setUpClass(cls): + def load_result(): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput = os.path.join(testInputs, 'test2_run.json') + with open(testOutput, 'r') as f: + json = json.load(f) + return json + + json = load_result() + json1 = filter_benchmark(json, "BM_Z.ro", ".") + json2 = filter_benchmark(json, "BM_O.e", ".") + cls.json_diff_report = get_difference_report(json1, json2) - def test_basic(self): + def test_json_diff_report_pretty_printing(self): expect_lines = [ ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'], ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'], ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'], ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'], ] - json = self.load_result() - json1 = filter_benchmark(json, "BM_Z.ro", ".") - json2 = filter_benchmark(json, "BM_O.e", ".") - output_lines_with_header = generate_difference_report( - json1, json2, use_color=False) + output_lines_with_header = print_difference_report( + self.json_diff_report, use_color=False) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) @@ -397,24 +537,64 @@ def test_basic(self): self.assertEqual(len(parts), 7) self.assertEqual(expect_lines[i], parts) + def test_json_diff_report(self): + expected_output = [ + { + 'name': u'.', + 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 10, 'real_time_other': 5, 'cpu_time': 10, 'cpu_time_other': 5}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': u'./4', + 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 40, 'real_time_other': 20, 'cpu_time': 40, 'cpu_time_other': 20}], + 'time_unit': 'ns', + 'utest': {}, + }, + { + 'name': u'Prefix/.', + 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 20, 'real_time_other': 10, 'cpu_time': 20, 'cpu_time_other': 10}], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': u'Prefix/./3', + 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}], + 'time_unit': 'ns', + 'utest': {} + } + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip( + self.json_diff_report, expected_output): + self.assertEqual(out['name'], expected['name']) + self.assertEqual(out['time_unit'], expected['time_unit']) + assert_utest(self, out, expected) + assert_measurements(self, out, expected) + class TestReportDifferenceWithUTest(unittest.TestCase): - def load_results(self): - import json - testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test3_run0.json') - testOutput2 = os.path.join(testInputs, 'test3_run1.json') - with open(testOutput1, 'r') as f: - json1 = json.load(f) - with open(testOutput2, 'r') as f: - json2 = json.load(f) - return json1, json2 - - def test_utest(self): - expect_lines = [] + @classmethod + def setUpClass(cls): + def load_results(): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput1 = os.path.join(testInputs, 'test3_run0.json') + testOutput2 = os.path.join(testInputs, 'test3_run1.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report( + json1, json2, utest=True) + + def test_json_diff_report_pretty_printing(self): expect_lines = [ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], @@ -453,9 +633,8 @@ def test_utest(self): 'recommended.'], ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], ] - json1, json2 = self.load_results() - output_lines_with_header = generate_difference_report( - json1, json2, utest=True, utest_alpha=0.05, use_color=False) + output_lines_with_header = print_difference_report( + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False) output_lines = output_lines_with_header[2:] print("\n") print("\n".join(output_lines_with_header)) @@ -464,25 +643,105 @@ def test_utest(self): parts = [x for x in output_lines[i].split(' ') if x] self.assertEqual(expect_lines[i], parts) + def test_json_diff_report(self): + expected_output = [ + { + 'name': u'BM_One', + 'measurements': [ + {'time': -0.1, + 'cpu': 0.1, + 'real_time': 10, + 'real_time_other': 9, + 'cpu_time': 100, + 'cpu_time_other': 110} + ], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': u'BM_Two', + 'measurements': [ + {'time': 0.1111111111111111, + 'cpu': -0.011111111111111112, + 'real_time': 9, + 'real_time_other': 10, + 'cpu_time': 90, + 'cpu_time_other': 89}, + {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, + 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} + ], + 'time_unit': 'ns', + 'utest': { + 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387 + } + }, + { + 'name': u'short', + 'measurements': [ + {'time': -0.125, + 'cpu': -0.0625, + 'real_time': 8, + 'real_time_other': 7, + 'cpu_time': 80, + 'cpu_time_other': 75}, + {'time': -0.4325, + 'cpu': -0.13506493506493514, + 'real_time': 8, + 'real_time_other': 4.54, + 'cpu_time': 77, + 'cpu_time_other': 66.6} + ], + 'time_unit': 'ns', + 'utest': { + 'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772 + } + }, + { + 'name': u'medium', + 'measurements': [ + {'time': -0.375, + 'cpu': -0.3375, + 'real_time': 8, + 'real_time_other': 5, + 'cpu_time': 80, + 'cpu_time_other': 53} + ], + 'time_unit': 'ns', + 'utest': {} + } + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip( + self.json_diff_report, expected_output): + self.assertEqual(out['name'], expected['name']) + self.assertEqual(out['time_unit'], expected['time_unit']) + assert_utest(self, out, expected) + assert_measurements(self, out, expected) + class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( unittest.TestCase): - def load_results(self): - import json - testInputs = os.path.join( - os.path.dirname( - os.path.realpath(__file__)), - 'Inputs') - testOutput1 = os.path.join(testInputs, 'test3_run0.json') - testOutput2 = os.path.join(testInputs, 'test3_run1.json') - with open(testOutput1, 'r') as f: - json1 = json.load(f) - with open(testOutput2, 'r') as f: - json2 = json.load(f) - return json1, json2 - - def test_utest(self): - expect_lines = [] + @classmethod + def setUpClass(cls): + def load_results(): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput1 = os.path.join(testInputs, 'test3_run0.json') + testOutput2 = os.path.join(testInputs, 'test3_run1.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report( + json1, json2, utest=True) + + def test_json_diff_report_pretty_printing(self): expect_lines = [ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], @@ -519,10 +778,10 @@ def test_utest(self): '9+', 'repetitions', 'recommended.'], + ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'] ] - json1, json2 = self.load_results() - output_lines_with_header = generate_difference_report( - json1, json2, display_aggregates_only=True, + output_lines_with_header = print_difference_report( + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False) output_lines = output_lines_with_header[2:] print("\n") @@ -532,6 +791,109 @@ def test_utest(self): parts = [x for x in output_lines[i].split(' ') if x] self.assertEqual(expect_lines[i], parts) + def test_json_diff_report(self): + expected_output = [ + { + 'name': u'BM_One', + 'measurements': [ + {'time': -0.1, + 'cpu': 0.1, + 'real_time': 10, + 'real_time_other': 9, + 'cpu_time': 100, + 'cpu_time_other': 110} + ], + 'time_unit': 'ns', + 'utest': {} + }, + { + 'name': u'BM_Two', + 'measurements': [ + {'time': 0.1111111111111111, + 'cpu': -0.011111111111111112, + 'real_time': 9, + 'real_time_other': 10, + 'cpu_time': 90, + 'cpu_time_other': 89}, + {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8, + 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72} + ], + 'time_unit': 'ns', + 'utest': { + 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387 + } + }, + { + 'name': u'short', + 'measurements': [ + {'time': -0.125, + 'cpu': -0.0625, + 'real_time': 8, + 'real_time_other': 7, + 'cpu_time': 80, + 'cpu_time_other': 75}, + {'time': -0.4325, + 'cpu': -0.13506493506493514, + 'real_time': 8, + 'real_time_other': 4.54, + 'cpu_time': 77, + 'cpu_time_other': 66.6} + ], + 'time_unit': 'ns', + 'utest': { + 'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772 + } + }, + { + 'name': u'medium', + 'measurements': [ + {'real_time_other': 5, + 'cpu_time': 80, + 'time': -0.375, + 'real_time': 8, + 'cpu_time_other': 53, + 'cpu': -0.3375 + } + ], + 'utest': {}, + 'time_unit': u'ns', + 'aggregate_name': '' + } + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip( + self.json_diff_report, expected_output): + self.assertEqual(out['name'], expected['name']) + self.assertEqual(out['time_unit'], expected['time_unit']) + assert_utest(self, out, expected) + assert_measurements(self, out, expected) + + +def assert_utest(unittest_instance, lhs, rhs): + if lhs['utest']: + unittest_instance.assertAlmostEqual( + lhs['utest']['cpu_pvalue'], + rhs['utest']['cpu_pvalue']) + unittest_instance.assertAlmostEqual( + lhs['utest']['time_pvalue'], + rhs['utest']['time_pvalue']) + unittest_instance.assertEqual( + lhs['utest']['have_optimal_repetitions'], + rhs['utest']['have_optimal_repetitions']) + else: + # lhs is empty. assert if rhs is not. + unittest_instance.assertEqual(lhs['utest'], rhs['utest']) + + +def assert_measurements(unittest_instance, lhs, rhs): + for m1, m2 in zip(lhs['measurements'], rhs['measurements']): + unittest_instance.assertEqual(m1['real_time'], m2['real_time']) + unittest_instance.assertEqual(m1['cpu_time'], m2['cpu_time']) + # m1['time'] and m1['cpu'] hold values which are being calculated, + # and therefore we must use almost-equal pattern. + unittest_instance.assertAlmostEqual(m1['time'], m2['time'], places=4) + unittest_instance.assertAlmostEqual(m1['cpu'], m2['cpu'], places=4) + if __name__ == '__main__': unittest.main() From 2d9bfe1ef80ad993d1af1c614d6f238b2ea0f802 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 28 Sep 2020 10:54:55 +0100 Subject: [PATCH 244/330] Include github workflows status in README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 41a1bdff75..6c09b9d76a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ # Benchmark +[![build-and-test](https://github.com/google/benchmark/workflows/build-and-test/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Abuild-and-test) +[![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) +[![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) + [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) -[![slackin](https://slackin-iqtfqnpzxd.now.sh/badge.svg)](https://slackin-iqtfqnpzxd.now.sh/) + A library to benchmark code snippets, similar to unit tests. Example: From ffe1342eb2faa7d2e7c35b4db2ccf99fab81ec20 Mon Sep 17 00:00:00 2001 From: Min-Yih Hsu Date: Mon, 28 Sep 2020 23:35:18 -0700 Subject: [PATCH 245/330] Add CycleTimer implementation for M68K architecture (#1050) As per discussions in here [1], LLVM is going to get backend support on Motorola 68000 series CPUs (a.k.a M68K or M680x0). So it's necessary to add CycleTimer implementation here, which is simply using `gettimeofday` same as MIPS. This fixes #1049 [1] https://reviews.llvm.org/D88389 --- AUTHORS | 1 + CONTRIBUTORS | 1 + src/cycleclock.h | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index e353b53bf3..3068b2ebff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -55,3 +55,4 @@ Stripe, Inc. Yixuan Qiu Yusuke Suzuki Zbigniew Skowron +Min-Yih Hsu diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 6beed7166e..7071d7bf11 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -77,3 +77,4 @@ Tom Madams Yixuan Qiu Yusuke Suzuki Zbigniew Skowron +Min-Yih Hsu diff --git a/src/cycleclock.h b/src/cycleclock.h index 179c67cd61..93d579a739 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -161,7 +161,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__mips__) +#elif defined(__mips__) || defined(__m68k__) // mips apparently only allows rdtsc for superusers, so we fall // back to gettimeofday. It's possible clock_gettime would be better. struct timeval tv; From af72911f2fe6b8114564614d2db17a449f8c4af0 Mon Sep 17 00:00:00 2001 From: Michael Neumann Date: Mon, 12 Oct 2020 22:41:49 +0200 Subject: [PATCH 246/330] Add support for DragonFly BSD (#1058) Without this commit, compilation fails on DragonFly with the following message: ``` /home/mneumann/Dev/benchmark.old/src/sysinfo.cc:446:2: error: #warning "HOST_NAME_MAX not defined. using 64" [-Werror=cpp] ^~~~~~~ ``` Also note that the sysctl is actually `hw.tsc_frequency` on DragonFly: ``` $ sysctl hw.tsc_frequency hw.tsc_frequency: 3498984022 ``` Tested on: ``` $ uname -a DragonFly box.localnet 5.9-DEVELOPMENT DragonFly v5.9.0.742.g4b29dd-DEVELOPMENT #5: Tue Aug 18 00:21:31 CEST 2020 ``` --- src/internal_macros.h | 2 ++ src/sysinfo.cc | 12 ++++++++---- src/timers.cc | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/internal_macros.h b/src/internal_macros.h index 6adf00d056..889b353724 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -58,6 +58,8 @@ #define BENCHMARK_OS_NETBSD 1 #elif defined(__OpenBSD__) #define BENCHMARK_OS_OPENBSD 1 +#elif defined(__DragonFly__) + #define BENCHMARK_OS_DRAGONFLY 1 #elif defined(__linux__) #define BENCHMARK_OS_LINUX 1 #elif defined(__native_client__) diff --git a/src/sysinfo.cc b/src/sysinfo.cc index 8bab9320f1..b30b4f832e 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -29,7 +29,8 @@ #include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include #if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX || \ - defined BENCHMARK_OS_NETBSD || defined BENCHMARK_OS_OPENBSD + defined BENCHMARK_OS_NETBSD || defined BENCHMARK_OS_OPENBSD || \ + defined BENCHMARK_OS_DRAGONFLY #define BENCHMARK_HAS_SYSCTL #include #endif @@ -607,6 +608,8 @@ double GetCPUCyclesPerSecond() { "machdep.tsc_freq"; #elif defined BENCHMARK_OS_OPENBSD "hw.cpuspeed"; +#elif defined BENCHMARK_OS_DRAGONFLY + "hw.tsc_frequency"; #else "hw.cpufrequency"; #endif @@ -671,9 +674,10 @@ double GetCPUCyclesPerSecond() { } std::vector GetLoadAvg() { -#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ - defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ - defined BENCHMARK_OS_OPENBSD) && !defined(__ANDROID__) +#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ + defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ + defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \ + !defined(__ANDROID__) constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); diff --git a/src/timers.cc b/src/timers.cc index 4f76eddc1d..1d3ab9a327 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -28,7 +28,8 @@ #include #include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD #include -#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX +#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_DRAGONFLY || \ + defined BENCHMARK_OS_MACOSX #include #endif #if defined(BENCHMARK_OS_MACOSX) From 3d1c2677686718d906f28c1d4da001c42666e6d2 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 15 Oct 2020 09:12:40 +0100 Subject: [PATCH 247/330] src/benchmark_register.h: add missing inclusion (#1060) Noticed missing header when was building llvm with gcc-11: ``` llvm-project/llvm/utils/benchmark/src/benchmark_register.h:17:30: error: 'numeric_limits' is not a member of 'std' 17 | static const T kmax = std::numeric_limits::max(); | ^~~~~~~~~~~~~~ ``` --- src/benchmark_register.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/benchmark_register.h b/src/benchmark_register.h index 61377d7423..204bf1d9f8 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -1,6 +1,7 @@ #ifndef BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H +#include #include #include "check.h" From dce3322a549650d18f50b5f1428a5942327ab6a5 Mon Sep 17 00:00:00 2001 From: Fanbo Meng Date: Wed, 21 Oct 2020 11:39:54 -0400 Subject: [PATCH 248/330] Add support for z/OS XL compiler inline asm syntax (#1063) On s390 architecture, z/OS XL compiler uses HLASM inline assembly, which has different syntax and needs to be distinguished to avoid compilation error. --- CONTRIBUTORS | 1 + src/cycleclock.h | 5 +++++ src/internal_macros.h | 8 +++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 7071d7bf11..d2f64356b4 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -41,6 +41,7 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fanbo Meng Federico Ficarelli Felix Homann Geoffrey Martin-Noble diff --git a/src/cycleclock.h b/src/cycleclock.h index 93d579a739..77be7b927b 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -170,7 +170,12 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #elif defined(__s390__) // Covers both s390 and s390x. // Return the CPU clock. uint64_t tsc; +#if defined(BENCHMARK_OS_ZOS) && defined(COMPILER_IBMXL) + // z/OS XL compiler HLASM syntax. + asm(" stck %0" : "=m"(tsc) : : "cc"); +#else asm("stck %0" : "=Q"(tsc) : : "cc"); +#endif return tsc; #elif defined(__riscv) // RISC-V // Use RDCYCLE (and RDCYCLEH on riscv32) diff --git a/src/internal_macros.h b/src/internal_macros.h index 889b353724..91f367b894 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -13,7 +13,11 @@ #endif #if defined(__clang__) - #if !defined(COMPILER_CLANG) + #if defined(__ibmxl__) + #if !defined(COMPILER_IBMXL) + #define COMPILER_IBMXL + #endif + #elif !defined(COMPILER_CLANG) #define COMPILER_CLANG #endif #elif defined(_MSC_VER) @@ -74,6 +78,8 @@ #define BENCHMARK_OS_SOLARIS 1 #elif defined(__QNX__) #define BENCHMARK_OS_QNX 1 +#elif defined(__MVS__) +#define BENCHMARK_OS_ZOS 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) From a9704c268dd2ed8bc65d8fc2880cb7a0ddd64d2c Mon Sep 17 00:00:00 2001 From: Abhina Sree <69635948+abhina-sree@users.noreply.github.com> Date: Thu, 29 Oct 2020 04:49:02 -0400 Subject: [PATCH 249/330] Nanosleep workaround for z/OS in sleep.cc (#1067) * z/OS does not support nanosleep, add workaround to use sleep() and usleep() instead * change unsigned to int, and fix while loop --- CONTRIBUTORS | 1 + src/sleep.cc | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d2f64356b4..802ce0dcfb 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -22,6 +22,7 @@ # # Please keep the list sorted. +Abhina Sreeskantharajan Albert Pretorius Alex Steele Andriy Berestovskyy diff --git a/src/sleep.cc b/src/sleep.cc index 1512ac90f7..4609d540ea 100644 --- a/src/sleep.cc +++ b/src/sleep.cc @@ -24,6 +24,10 @@ #include #endif +#ifdef BENCHMARK_OS_ZOS +#include +#endif + namespace benchmark { #ifdef BENCHMARK_OS_WINDOWS // Window's Sleep takes milliseconds argument. @@ -33,11 +37,23 @@ void SleepForSeconds(double seconds) { } #else // BENCHMARK_OS_WINDOWS void SleepForMicroseconds(int microseconds) { +#ifdef BENCHMARK_OS_ZOS + // z/OS does not support nanosleep. Instead call sleep() and then usleep() to + // sleep for the remaining microseconds because usleep() will fail if its + // argument is greater than 1000000. + div_t sleepTime = div(microseconds, kNumMicrosPerSecond); + int seconds = sleepTime.quot; + while (seconds != 0) + seconds = sleep(seconds); + while (usleep(sleepTime.rem) == -1 && errno == EINTR) + ; +#else struct timespec sleep_time; sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) ; // Ignore signals and wait for the full interval to elapse. +#endif } void SleepForMilliseconds(int milliseconds) { From d9abf017632be4a00b92cf4289539b353fcea5d2 Mon Sep 17 00:00:00 2001 From: Steven Wan Date: Tue, 3 Nov 2020 04:08:46 -0500 Subject: [PATCH 250/330] Rename 'mftbl' to 'mftb' (#1069) * Rename 'mftbl' to 'mftb' * Add my name to the contributor list --- CONTRIBUTORS | 1 + src/cycleclock.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 802ce0dcfb..b5e1aa4fd1 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -74,6 +74,7 @@ Robert Guo Roman Lebedev Sayan Bhattacharjee Shuo Chen +Steven Wan Tobias Ulvgård Tom Madams Yixuan Qiu diff --git a/src/cycleclock.h b/src/cycleclock.h index 77be7b927b..89de86fa20 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -92,7 +92,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { uint32_t tbl, tbu0, tbu1; asm volatile( "mftbu %0\n" - "mftbl %1\n" + "mftb %1\n" "mftbu %2" : "=r"(tbu0), "=r"(tbl), "=r"(tbu1)); tbl &= -static_cast(tbu0 == tbu1); From 348aa2c964494b5947c0e7f96b82c1fe844d684f Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 6 Nov 2020 12:10:04 +0300 Subject: [PATCH 251/330] bazel support for tools (#982) * packages versions updated to be in sync with modern python versions --- WORKSPACE | 13 +++++++++++++ requirements.txt | 2 ++ tools/BUILD.bazel | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 requirements.txt create mode 100644 tools/BUILD.bazel diff --git a/WORKSPACE b/WORKSPACE index c00d12cd17..fdc8c7f8e4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -34,3 +34,16 @@ new_local_repository( build_file = "@//bindings/python:python_headers.BUILD", path = "/usr/include/python3.6", # May be overwritten by setup.py. ) + +http_archive( + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +) + +load("@rules_python//python:pip.bzl", pip3_install="pip_install") + +pip3_install( + name = "py_deps", + requirements = "//:requirements.txt", +) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..85e8986040 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +numpy == 1.19.4 +scipy == 1.5.4 diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel new file mode 100644 index 0000000000..5895883a2e --- /dev/null +++ b/tools/BUILD.bazel @@ -0,0 +1,19 @@ +load("@py_deps//:requirements.bzl", "requirement") + +py_library( + name = "gbench", + srcs = glob(["gbench/*.py"]), + deps = [ + requirement("numpy"), + requirement("scipy"), + ], +) + +py_binary( + name = "compare", + srcs = ["compare.py"], + python_version = "PY2", + deps = [ + ":gbench", + ], +) From 37ced31bfc9fe09d1418fb197e420408eb8c86bf Mon Sep 17 00:00:00 2001 From: Mario Emmenlauer Date: Thu, 19 Nov 2020 14:50:30 +0100 Subject: [PATCH 252/330] Added support for macro expansion in benchmark names (#1054) * Adding test for defined names in test fixtures * include/benchmark/benchmark.h: Added support for macro expansion in benchmark names --- include/benchmark/benchmark.h | 21 ++++++++++++--------- test/fixture_test.cc | 14 ++++++++------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 01f12620ee..834687e640 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1106,6 +1106,9 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_CONCAT(_benchmark_, BENCHMARK_PRIVATE_UNIQUE_ID, n) #define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) #define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c +// Helper for concatenation with macro name expansion +#define BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method) \ + BaseClass##_##Method##_Benchmark #define BENCHMARK_PRIVATE_DECLARE(n) \ static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ @@ -1226,27 +1229,27 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_DEFINE_F(BaseClass, Method) \ BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #define BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) \ BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #define BENCHMARK_TEMPLATE2_DEFINE_F(BaseClass, Method, a, b) \ BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #ifdef BENCHMARK_HAS_CXX11 #define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, ...) \ BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #else #define BENCHMARK_TEMPLATE_DEFINE_F(BaseClass, Method, a) \ BENCHMARK_TEMPLATE1_DEFINE_F(BaseClass, Method, a) #endif #define BENCHMARK_REGISTER_F(BaseClass, Method) \ - BENCHMARK_PRIVATE_REGISTER_F(BaseClass##_##Method##_Benchmark) + BENCHMARK_PRIVATE_REGISTER_F(BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)) #define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ BENCHMARK_PRIVATE_DECLARE(TestName) = \ @@ -1256,23 +1259,23 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_F(BaseClass, Method) \ BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #define BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) \ BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #define BENCHMARK_TEMPLATE2_F(BaseClass, Method, a, b) \ BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #ifdef BENCHMARK_HAS_CXX11 #define BENCHMARK_TEMPLATE_F(BaseClass, Method, ...) \ BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(BaseClass, Method, __VA_ARGS__) \ BENCHMARK_REGISTER_F(BaseClass, Method); \ - void BaseClass##_##Method##_Benchmark::BenchmarkCase + void BENCHMARK_PRIVATE_CONCAT_NAME(BaseClass, Method)::BenchmarkCase #else #define BENCHMARK_TEMPLATE_F(BaseClass, Method, a) \ BENCHMARK_TEMPLATE1_F(BaseClass, Method, a) diff --git a/test/fixture_test.cc b/test/fixture_test.cc index 1462b10f02..a331c7de8d 100644 --- a/test/fixture_test.cc +++ b/test/fixture_test.cc @@ -4,7 +4,9 @@ #include #include -class MyFixture : public ::benchmark::Fixture { +#define FIXTURE_BECHMARK_NAME MyFixture + +class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: void SetUp(const ::benchmark::State& state) { if (state.thread_index == 0) { @@ -20,19 +22,19 @@ class MyFixture : public ::benchmark::Fixture { } } - ~MyFixture() { assert(data == nullptr); } + ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } std::unique_ptr data; }; -BENCHMARK_F(MyFixture, Foo)(benchmark::State &st) { +BENCHMARK_F(FIXTURE_BECHMARK_NAME, Foo)(benchmark::State &st) { assert(data.get() != nullptr); assert(*data == 42); for (auto _ : st) { } } -BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) { +BENCHMARK_DEFINE_F(FIXTURE_BECHMARK_NAME, Bar)(benchmark::State& st) { if (st.thread_index == 0) { assert(data.get() != nullptr); assert(*data == 42); @@ -43,7 +45,7 @@ BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) { } st.SetItemsProcessed(st.range(0)); } -BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42); -BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42)->ThreadPerCpu(); +BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42); +BENCHMARK_REGISTER_F(FIXTURE_BECHMARK_NAME, Bar)->Arg(42)->ThreadPerCpu(); BENCHMARK_MAIN(); From 7fa6f1f91a96d8763de754810710d457f7fea575 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 23 Nov 2020 10:02:33 +0000 Subject: [PATCH 253/330] Disable lto-type-mismatch warnings (#1071) --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a157666148..1007254520 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -194,6 +194,7 @@ else() # Link time optimisation if (BENCHMARK_ENABLE_LTO) add_cxx_compiler_flag(-flto) + add_cxx_compiler_flag(-Wno-lto-type-mismatch) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") find_program(GCC_AR gcc-ar) if (GCC_AR) From 17a6b21ee15c71c5570d6724920cf3289b5d88ab Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 26 Nov 2020 03:12:45 -0800 Subject: [PATCH 254/330] Fix Range when starting at zero (#1073) The existing behavior results in the `0` value being added twice. Since `lo` is always added to `dst`, we never want to explicitly add `0` if `lo` is equal to `0`. --- src/benchmark_register.h | 2 +- test/benchmark_gtest.cc | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/benchmark_register.h b/src/benchmark_register.h index 204bf1d9f8..c774e6f8ea 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -87,7 +87,7 @@ void AddRange(std::vector* dst, T lo, T hi, int mult) { } // Treat 0 as a special case (see discussion on #762). - if (lo <= 0 && hi >= 0) { + if (lo < 0 && hi >= 0) { dst->push_back(0); } diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index 9557b20ec7..6dbf7a5042 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -90,6 +90,12 @@ TEST(AddRangeTest, ZeroOnlyRange) { EXPECT_THAT(dst, testing::ElementsAre(0)); } +TEST(AddRangeTest, ZeroStartingRange) { + std::vector dst; + AddRange(&dst, 0, 2, 2); + EXPECT_THAT(dst, testing::ElementsAre(0, 1, 2)); +} + TEST(AddRangeTest, NegativeRange64) { std::vector dst; AddRange(&dst, -4, 4, 2); From bf585a2789e30585b4e3ce6baf11ef2750b54677 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 26 Nov 2020 11:13:23 +0000 Subject: [PATCH 255/330] Fix some bazel warnings about missing sha256 on http_archives --- WORKSPACE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index fdc8c7f8e4..631f3ba05d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -6,6 +6,7 @@ http_archive( name = "rules_cc", strip_prefix = "rules_cc-a508235df92e71d537fcbae0c7c952ea6957a912", urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"], + sha256 = "d7dc12c1d5bc1a87474de8e3d17b7731a4dcebcfb8aa3990fe8ac7734ef12f2f", ) http_archive( @@ -19,6 +20,7 @@ http_archive( name = "com_google_googletest", strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e", urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"], + sha256 = "8f827dd550db8b4fdf73904690df0be9fccc161017c9038a724bc9a0617a1bc8", ) http_archive( From 378ed8ff25c01fe55c35cad6ed1eeab02c15d49a Mon Sep 17 00:00:00 2001 From: feserr Date: Mon, 21 Dec 2020 18:15:58 +0100 Subject: [PATCH 256/330] Add 'seconds' time unit (#1076) Fixes #1075. * Add an option to report in seconds. * Reduce the time of the test. * Add CSV/JSON tests for new time reports. --- bindings/python/google_benchmark/__init__.py | 2 + bindings/python/google_benchmark/benchmark.cc | 1 + include/benchmark/benchmark.h | 6 +- test/options_test.cc | 1 + test/output_test_helper.cc | 5 ++ test/reporter_output_test.cc | 89 ++++++++++++++++++- 6 files changed, 102 insertions(+), 2 deletions(-) diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index 787c423d5d..f31285eb76 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -34,6 +34,7 @@ def my_benchmark(state): kNanosecond, kMicrosecond, kMillisecond, + kSecond, oNone, o1, oN, @@ -53,6 +54,7 @@ def my_benchmark(state): "kNanosecond", "kMicrosecond", "kMillisecond", + "kSecond", "oNone", "o1", "oN", diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index a733339769..d80816e5d4 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -49,6 +49,7 @@ PYBIND11_MODULE(_benchmark, m) { .value("kNanosecond", TimeUnit::kNanosecond) .value("kMicrosecond", TimeUnit::kMicrosecond) .value("kMillisecond", TimeUnit::kMillisecond) + .value("kSecond", TimeUnit::kSecond) .export_values(); using benchmark::BigO; diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 834687e640..f57e3e79bd 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -407,7 +407,7 @@ typedef std::map UserCounters; // TimeUnit is passed to a benchmark in order to specify the order of magnitude // for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; // BigO is passed to a benchmark in order to specify the asymptotic // computational @@ -1577,6 +1577,8 @@ class MemoryManager { inline const char* GetTimeUnitString(TimeUnit unit) { switch (unit) { + case kSecond: + return "s"; case kMillisecond: return "ms"; case kMicrosecond: @@ -1589,6 +1591,8 @@ inline const char* GetTimeUnitString(TimeUnit unit) { inline double GetTimeUnitMultiplier(TimeUnit unit) { switch (unit) { + case kSecond: + return 1; case kMillisecond: return 1e3; case kMicrosecond: diff --git a/test/options_test.cc b/test/options_test.cc index 7bfc235465..9f9a78667c 100644 --- a/test/options_test.cc +++ b/test/options_test.cc @@ -25,6 +25,7 @@ BENCHMARK(BM_basic)->Arg(42); BENCHMARK(BM_basic_slow)->Arg(10)->Unit(benchmark::kNanosecond); BENCHMARK(BM_basic_slow)->Arg(100)->Unit(benchmark::kMicrosecond); BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kSecond); BENCHMARK(BM_basic)->Range(1, 8); BENCHMARK(BM_basic)->RangeMultiplier(2)->Range(1, 8); BENCHMARK(BM_basic)->DenseRange(10, 15); diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index f99b3a8261..1aebc55d01 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -48,6 +48,9 @@ SubMap& GetSubstitutions() { {" %s ", "[ ]+"}, {"%time", "[ ]*" + time_re + "[ ]+ns"}, {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, + {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, + {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"}, + {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"}, {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, @@ -56,6 +59,8 @@ SubMap& GetSubstitutions() { "items_per_second,label,error_occurred,error_message"}, {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, + {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, + {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index bcce007831..d24a57dda3 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -168,6 +168,93 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " "label\"%csv_label_report_end$"}}); +// ========================================================================= // +// ------------------------ Testing Time Label Output ---------------------- // +// ========================================================================= // + +void BM_time_label_nanosecond(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond); + +ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_time_label_nanosecond\",$"}, + {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}}); + +void BM_time_label_microsecond(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond); + +ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_time_label_microsecond\",$"}, + {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"us\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}}); + +void BM_time_label_millisecond(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond); + +ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_time_label_millisecond\",$"}, + {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ms\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}}); + +void BM_time_label_second(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond); + +ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"}, + {"\"run_name\": \"BM_time_label_second\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"s\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}}); + // ========================================================================= // // ------------------------ Testing Error Output --------------------------- // // ========================================================================= // @@ -712,7 +799,7 @@ ADD_CASES( // ========================================================================= // // ------------------------- Testing StrEscape JSON ------------------------ // // ========================================================================= // -#if 0 // enable when csv testing code correctly handles multi-line fields +#if 0 // enable when csv testing code correctly handles multi-line fields void BM_JSON_Format(benchmark::State& state) { state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes"); for (auto _ : state) { From d8254bb9eb5f6deeddee639d0b27347e186e0a84 Mon Sep 17 00:00:00 2001 From: Aidan Wolter Date: Tue, 22 Dec 2020 01:54:33 -0800 Subject: [PATCH 257/330] Add bazel target for benchmark_release (#1078) Fixes google#1077 Bazel clients currently cannot build the benchmark library in Release mode. This commit adds a new target ":benchmark_release" to enable this. --- BUILD.bazel | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index eb35b62730..2d87177c86 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,8 +10,8 @@ config_setting( visibility = [":__subpackages__"], ) -cc_library( - name = "benchmark", +filegroup( + name = "benchmark_srcs", srcs = glob( [ "src/*.cc", @@ -19,7 +19,25 @@ cc_library( ], exclude = ["src/benchmark_main.cc"], ), +) + +cc_library( + name = "benchmark", + srcs = [":benchmark_srcs"], + hdrs = ["include/benchmark/benchmark.h"], + linkopts = select({ + ":windows": ["-DEFAULTLIB:shlwapi.lib"], + "//conditions:default": ["-pthread"], + }), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) + +cc_library( + name = "benchmark_release", + srcs = [":benchmark_srcs"], hdrs = ["include/benchmark/benchmark.h"], + defines = ["NDEBUG"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], "//conditions:default": ["-pthread"], From a6d08aea4b70c5532736924377df8be62ef2067a Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 22 Dec 2020 11:47:52 +0000 Subject: [PATCH 258/330] Create workflow to exercise bazel build (#1079) * Create workflow to exercise bazel build --- .github/workflows/bazel.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/bazel.yml diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml new file mode 100644 index 0000000000..d6bbe62905 --- /dev/null +++ b/.github/workflows/bazel.yml @@ -0,0 +1,33 @@ +name: bazel + +on: + push: {} + pull_request: {} + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: mount bazel cache + uses: actions/cache@v1 + with: + path: "/home/runner/.cache/bazel" + key: bazel + + - name: install bazelisk + run: | + curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64" + mkdir -p "${GITHUB_WORKSPACE}/bin/" + mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" + chmod +x "${GITHUB_WORKSPACE}/bin/bazel" + + - name: build + run: | + "${GITHUB_WORKSPACE}/bin/bazel" build //... + + - name: test + run: | + "${GITHUB_WORKSPACE}/bin/bazel" test //test/... From 8df87f6c879cbcabd17c5cfcec7b89687df36953 Mon Sep 17 00:00:00 2001 From: Yannic Date: Tue, 5 Jan 2021 10:54:04 +0100 Subject: [PATCH 259/330] Revert "Add bazel target for benchmark_release (#1078)" (#1081) This reverts commit d8254bb9eb5f6deeddee639d0b27347e186e0a84. --- BUILD.bazel | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 2d87177c86..eb35b62730 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,8 +10,8 @@ config_setting( visibility = [":__subpackages__"], ) -filegroup( - name = "benchmark_srcs", +cc_library( + name = "benchmark", srcs = glob( [ "src/*.cc", @@ -19,25 +19,7 @@ filegroup( ], exclude = ["src/benchmark_main.cc"], ), -) - -cc_library( - name = "benchmark", - srcs = [":benchmark_srcs"], - hdrs = ["include/benchmark/benchmark.h"], - linkopts = select({ - ":windows": ["-DEFAULTLIB:shlwapi.lib"], - "//conditions:default": ["-pthread"], - }), - strip_include_prefix = "include", - visibility = ["//visibility:public"], -) - -cc_library( - name = "benchmark_release", - srcs = [":benchmark_srcs"], hdrs = ["include/benchmark/benchmark.h"], - defines = ["NDEBUG"], linkopts = select({ ":windows": ["-DEFAULTLIB:shlwapi.lib"], "//conditions:default": ["-pthread"], From ea5a5bbff491fd625c6e3458f6edd680b8bd5452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Janiszewski?= Date: Fri, 12 Feb 2021 11:30:26 +0100 Subject: [PATCH 260/330] Add MSVC ARM64 support for reading clocks (#1052) Lacks CMake support, see https://github.com/google/benchmark/pull/1090 --- src/cycleclock.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cycleclock.h b/src/cycleclock.h index 89de86fa20..6843b69bd4 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -36,7 +36,7 @@ // declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(COMPILER_MSVC) && !defined(_M_IX86) +#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif @@ -114,6 +114,12 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // when I know it will work. Otherwise, I'll use __rdtsc and hope // the code is being compiled with a non-ancient compiler. _asm rdtsc +#elif defined(COMPILER_MSVC) && defined(_M_ARM64) + // See https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=vs-2019 + // and https://reviews.llvm.org/D53115 + int64_t virtual_timer_value; + virtual_timer_value = _ReadStatusReg(ARM64_CNTVCT); + return virtual_timer_value; #elif defined(COMPILER_MSVC) return __rdtsc(); #elif defined(BENCHMARK_OS_NACL) From d90321ff7ab553f97449b7017caf29ec18e74c6b Mon Sep 17 00:00:00 2001 From: SSE4 Date: Sun, 14 Feb 2021 06:45:57 -0800 Subject: [PATCH 261/330] - add support for Elbrus 2000 (e2k) (#1091) Signed-off-by: SSE4 --- src/cycleclock.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cycleclock.h b/src/cycleclock.h index 6843b69bd4..9bef594bed 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -204,6 +204,10 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { asm volatile("rdcycle %0" : "=r"(cycles)); return cycles; #endif +#elif defined(__e2k__) || defined(__elbrus__) + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #else // The soft failover to a generic implementation is automatic only for ARM. // For other platforms the developer is expected to make an attempt to create From a9b9471c0290c53a038150da607b8c054bdc574d Mon Sep 17 00:00:00 2001 From: Phoenix Meadowlark Date: Mon, 22 Feb 2021 01:55:07 -0800 Subject: [PATCH 262/330] Fix typo in invalid file name error message. (#1094) --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 1c049f2884..ffe4bf45a6 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -377,7 +377,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, if (!fname.empty()) { output_file.open(fname); if (!output_file.is_open()) { - Err << "invalid file name: '" << fname << std::endl; + Err << "invalid file name: '" << fname << "'" << std::endl; std::exit(1); } if (!file_reporter) { From 5c43112eb3d70cec10784d8d1f4373bc7397836b Mon Sep 17 00:00:00 2001 From: Phoenix Meadowlark Date: Mon, 22 Feb 2021 01:55:59 -0800 Subject: [PATCH 263/330] Update 'Output Files' section to reflect csv support. (#1095) --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c09b9d76a..7cf7f4a0ab 100644 --- a/README.md +++ b/README.md @@ -391,8 +391,12 @@ name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label Write benchmark results to a file with the `--benchmark_out=` option (or set `BENCHMARK_OUT`). Specify the output format with `--benchmark_out_format={json|console|csv}` (or set -`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that specifying -`--benchmark_out` does not suppress the console output. +`BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is +deperecated and the saved `.csv` file +[is not parsable](https://github.com/google/benchmark/issues/794) by csv +parsers. + +Specifying `--benchmark_out` does not suppress the console output. From 4c26070de053c315ebe7a9c82dfb8cc9eda625d0 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 5 Mar 2021 15:04:18 +0000 Subject: [PATCH 264/330] disable bazel for now (#1101) Unclear why this is failing so disabling for now. bazel build still works locally so this is a bazelisk/github workflow thing. --- .github/workflows/bazel.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index d6bbe62905..182e9040de 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -11,23 +11,23 @@ jobs: steps: - uses: actions/checkout@v1 - - name: mount bazel cache - uses: actions/cache@v1 - with: - path: "/home/runner/.cache/bazel" - key: bazel +# - name: mount bazel cache +# uses: actions/cache@v1 +# with: +# path: "/home/runner/.cache/bazel" +# key: bazel - - name: install bazelisk - run: | - curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64" - mkdir -p "${GITHUB_WORKSPACE}/bin/" - mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" - chmod +x "${GITHUB_WORKSPACE}/bin/bazel" +# - name: install bazelisk +# run: | +# curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64" +# mkdir -p "${GITHUB_WORKSPACE}/bin/" +# mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" +# chmod +x "${GITHUB_WORKSPACE}/bin/bazel" - - name: build - run: | - "${GITHUB_WORKSPACE}/bin/bazel" build //... +# - name: build +# run: | +# "${GITHUB_WORKSPACE}/bin/bazel" build //... - - name: test - run: | - "${GITHUB_WORKSPACE}/bin/bazel" test //test/... +# - name: test +# run: | +# "${GITHUB_WORKSPACE}/bin/bazel" test //test/... From 50c9eb54962c5adaa9c6804848b96216fb66c0ad Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 5 Mar 2021 16:37:55 +0000 Subject: [PATCH 265/330] Removing conanfile (and support) per #1088 (#1099) * Removing conanfile (and support) per #1088 --- conan/CMakeLists.txt | 7 --- conan/test_package/CMakeLists.txt | 10 ---- conan/test_package/conanfile.py | 19 ------- conan/test_package/test_package.cpp | 18 ------- conanfile.py | 79 ----------------------------- 5 files changed, 133 deletions(-) delete mode 100644 conan/CMakeLists.txt delete mode 100644 conan/test_package/CMakeLists.txt delete mode 100644 conan/test_package/conanfile.py delete mode 100644 conan/test_package/test_package.cpp delete mode 100644 conanfile.py diff --git a/conan/CMakeLists.txt b/conan/CMakeLists.txt deleted file mode 100644 index 15b92ca91a..0000000000 --- a/conan/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -cmake_minimum_required(VERSION 2.8.11) -project(cmake_wrapper) - -include(conanbuildinfo.cmake) -conan_basic_setup() - -include(${CMAKE_SOURCE_DIR}/CMakeListsOriginal.txt) diff --git a/conan/test_package/CMakeLists.txt b/conan/test_package/CMakeLists.txt deleted file mode 100644 index 089a6c729d..0000000000 --- a/conan/test_package/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -cmake_minimum_required(VERSION 2.8.11) -project(test_package) - -set(CMAKE_VERBOSE_MAKEFILE TRUE) - -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_basic_setup() - -add_executable(${PROJECT_NAME} test_package.cpp) -target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) diff --git a/conan/test_package/conanfile.py b/conan/test_package/conanfile.py deleted file mode 100644 index d63f4088c9..0000000000 --- a/conan/test_package/conanfile.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from conans import ConanFile, CMake -import os - - -class TestPackageConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "cmake" - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - - def test(self): - bin_path = os.path.join("bin", "test_package") - self.run(bin_path, run_environment=True) diff --git a/conan/test_package/test_package.cpp b/conan/test_package/test_package.cpp deleted file mode 100644 index 4fa7ec0bf9..0000000000 --- a/conan/test_package/test_package.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "benchmark/benchmark.h" - -void BM_StringCreation(benchmark::State& state) { - while (state.KeepRunning()) - std::string empty_string; -} - -BENCHMARK(BM_StringCreation); - -void BM_StringCopy(benchmark::State& state) { - std::string x = "hello"; - while (state.KeepRunning()) - std::string copy(x); -} - -BENCHMARK(BM_StringCopy); - -BENCHMARK_MAIN(); diff --git a/conanfile.py b/conanfile.py deleted file mode 100644 index e31fc5268a..0000000000 --- a/conanfile.py +++ /dev/null @@ -1,79 +0,0 @@ -from conans import ConanFile, CMake, tools -from conans.errors import ConanInvalidConfiguration -import shutil -import os - - -class GoogleBenchmarkConan(ConanFile): - name = "benchmark" - description = "A microbenchmark support library." - topics = ("conan", "benchmark", "google", "microbenchmark") - url = "https://github.com/google/benchmark" - homepage = "https://github.com/google/benchmark" - author = "Google Inc." - license = "Apache-2.0" - exports_sources = ["*"] - generators = "cmake" - - settings = "arch", "build_type", "compiler", "os" - options = { - "shared": [True, False], - "fPIC": [True, False], - "enable_lto": [True, False], - "enable_exceptions": [True, False] - } - default_options = {"shared": False, "fPIC": True, "enable_lto": False, "enable_exceptions": True} - - _build_subfolder = "." - - def source(self): - # Wrap the original CMake file to call conan_basic_setup - shutil.move("CMakeLists.txt", "CMakeListsOriginal.txt") - shutil.move(os.path.join("conan", "CMakeLists.txt"), "CMakeLists.txt") - - def config_options(self): - if self.settings.os == "Windows": - if self.settings.compiler == "Visual Studio" and float(self.settings.compiler.version.value) <= 12: - raise ConanInvalidConfiguration("{} {} does not support Visual Studio <= 12".format(self.name, self.version)) - del self.options.fPIC - - def configure(self): - if self.settings.os == "Windows" and self.options.shared: - raise ConanInvalidConfiguration("Windows shared builds are not supported right now, see issue #639") - - def _configure_cmake(self): - cmake = CMake(self) - - cmake.definitions["BENCHMARK_ENABLE_TESTING"] = "OFF" - cmake.definitions["BENCHMARK_ENABLE_GTEST_TESTS"] = "OFF" - cmake.definitions["BENCHMARK_ENABLE_LTO"] = "ON" if self.options.enable_lto else "OFF" - cmake.definitions["BENCHMARK_ENABLE_EXCEPTIONS"] = "ON" if self.options.enable_exceptions else "OFF" - - # See https://github.com/google/benchmark/pull/638 for Windows 32 build explanation - if self.settings.os != "Windows": - cmake.definitions["BENCHMARK_BUILD_32_BITS"] = "ON" if "64" not in str(self.settings.arch) else "OFF" - cmake.definitions["BENCHMARK_USE_LIBCXX"] = "ON" if (str(self.settings.compiler.libcxx) == "libc++") else "OFF" - else: - cmake.definitions["BENCHMARK_USE_LIBCXX"] = "OFF" - - cmake.configure(build_folder=self._build_subfolder) - return cmake - - def build(self): - cmake = self._configure_cmake() - cmake.build() - - def package(self): - cmake = self._configure_cmake() - cmake.install() - - self.copy(pattern="LICENSE", dst="licenses") - - def package_info(self): - self.cpp_info.libs = tools.collect_libs(self) - if self.settings.os == "Linux": - self.cpp_info.libs.extend(["pthread", "rt"]) - elif self.settings.os == "Windows": - self.cpp_info.libs.append("shlwapi") - elif self.settings.os == "SunOS": - self.cpp_info.libs.append("kstat") From cc9abfc8f12577ea83b2d093693ba70c3c0fd2c7 Mon Sep 17 00:00:00 2001 From: Andre Meyering Date: Mon, 8 Mar 2021 10:23:24 +0100 Subject: [PATCH 266/330] Fix URL to googletest primer (#1102) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cf7f4a0ab..43b77ca7ec 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ To get started, see [Requirements](#requirements) and [Installation](#installation). See [Usage](#usage) for a full example and the [User Guide](#user-guide) for a more comprehensive feature overview. -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md) +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) as some of the structural aspects of the APIs are similar. ### Resources From 5e387e7d33a55b8d6b7c5025379b97cc9418fabf Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Tue, 30 Mar 2021 15:43:03 +0200 Subject: [PATCH 267/330] Implement custom benchmark name (#1107) * Implement custom benchmark name The benchmark's name can be changed using the Name() function which internally uses SetName(). * Update AUTHORS and CONTRIBUTORS * Describe new feature in README * Move new name function up Fixes #1106 --- AUTHORS | 1 + CONTRIBUTORS | 1 + README.md | 15 +++++++++++++++ include/benchmark/benchmark.h | 3 +++ src/benchmark_register.cc | 5 +++++ test/reporter_output_test.cc | 24 ++++++++++++++++++++++++ 6 files changed, 49 insertions(+) diff --git a/AUTHORS b/AUTHORS index 3068b2ebff..9b980419f2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -52,6 +52,7 @@ Sayan Bhattacharjee Shuo Chen Steinar H. Gunderson Stripe, Inc. +Tobias Schmidt Yixuan Qiu Yusuke Suzuki Zbigniew Skowron diff --git a/CONTRIBUTORS b/CONTRIBUTORS index b5e1aa4fd1..8370a2d737 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -75,6 +75,7 @@ Roman Lebedev Sayan Bhattacharjee Shuo Chen Steven Wan +Tobias Schmidt Tobias Ulvgård Tom Madams Yixuan Qiu diff --git a/README.md b/README.md index 43b77ca7ec..0d827b8dae 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,8 @@ too (`-lkstat`). [Passing Arguments](#passing-arguments) +[Custom Benchmark Name](#custom-benchmark-name) + [Calculating Asymptotic Complexity](#asymptotic-complexity) [Templated Benchmarks](#templated-benchmarks) @@ -652,6 +654,19 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2) ->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; }); ``` + + +### Custom Benchmark Name + +You can change the benchmark's name as follows: + +```c++ +BENCHMARK(BM_memcpy)->Name("memcpy")->RangeMultiplier(2)->Range(8, 8<<10); +``` + +The invocation will execute the benchmark as before using `BM_memcpy` but changes +the prefix in the report to `memcpy`. + ### Templated Benchmarks diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index f57e3e79bd..2e714b6eb0 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -790,6 +790,9 @@ class Benchmark { // Note: the following methods all return "this" so that multiple // method calls can be chained together in one expression. + // Specify the name of the benchmark + Benchmark* Name(const std::string& name); + // Run this benchmark once with "x" as the extra argument passed // to the function. // REQUIRES: The function passed to the constructor must accept an arg1. diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 65d9944f4f..c007876f24 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -278,6 +278,11 @@ Benchmark::Benchmark(const char* name) Benchmark::~Benchmark() {} +Benchmark* Benchmark::Name(const std::string& name) { + SetName(name.c_str()); + return this; +} + Benchmark* Benchmark::Arg(int64_t x) { CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); args_.push_back({x}); diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index d24a57dda3..b3d8edbae0 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -334,6 +334,30 @@ ADD_CASES(TC_JSONOut, {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); +// ========================================================================= // +// ------------------------ Testing Name Output ---------------------------- // +// ========================================================================= // + +void BM_name(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_name)->Name("BM_custom_name"); + +ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"}, + {"\"run_name\": \"BM_custom_name\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 0,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}}); + // ========================================================================= // // ------------------------ Testing Big Args Output ------------------------ // // ========================================================================= // From b8084e5054d2de304d4f3031f323265997790017 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 9 Apr 2021 12:59:31 +0100 Subject: [PATCH 268/330] fix minor typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d827b8dae..50cb89bf5f 100644 --- a/README.md +++ b/README.md @@ -394,7 +394,7 @@ Write benchmark results to a file with the `--benchmark_out=` option (or set `BENCHMARK_OUT`). Specify the output format with `--benchmark_out_format={json|console|csv}` (or set `BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is -deperecated and the saved `.csv` file +deprecated and the saved `.csv` file [is not parsable](https://github.com/google/benchmark/issues/794) by csv parsers. From 5a77a6d8dc9815d5bb0dd9f08b4f9e25b9b2274d Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 9 Apr 2021 13:00:06 +0100 Subject: [PATCH 269/330] fix minor typo --- bindings/python/google_benchmark/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/google_benchmark/__init__.py b/bindings/python/google_benchmark/__init__.py index f31285eb76..1055bf2418 100644 --- a/bindings/python/google_benchmark/__init__.py +++ b/bindings/python/google_benchmark/__init__.py @@ -110,7 +110,7 @@ def __decorator(func_or_options): # Alias for nicer API. -# We have to instanciate an object, even if stateless, to be able to use __getattr__ +# We have to instantiate an object, even if stateless, to be able to use __getattr__ # on option.range option = __OptionMaker() From f1deaf16b81ff2ead7a9df8c6fdaf5b54489d655 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 9 Apr 2021 13:00:43 +0100 Subject: [PATCH 270/330] fix minor typo (though this is an external property) --- bindings/python/google_benchmark/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index d80816e5d4..1b01fe7f7f 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -157,7 +157,7 @@ PYBIND11_MODULE(_benchmark, m) { .def("pause_timing", &State::PauseTiming) .def("resume_timing", &State::ResumeTiming) .def("skip_with_error", &State::SkipWithError) - .def_property_readonly("error_occured", &State::error_occurred) + .def_property_readonly("error_occurred", &State::error_occurred) .def("set_iteration_time", &State::SetIterationTime) .def_property("bytes_processed", &State::bytes_processed, &State::SetBytesProcessed) From 07578d82e0a4f99bdd1546f4f6f1727109f9420d Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Fri, 9 Apr 2021 12:32:00 -0400 Subject: [PATCH 271/330] Shrink the tz_offset size to 41. (#1110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building with gcc TSan on, and in Debug mode, we see a warning like: benchmark/src/timers.cc: In function ‘std::string benchmark::LocalDateTimeString()’: src/timers.cc:241:15: warning: ‘char* strncat(char*, const char*, size_t)’ output may be truncated copying 108 bytes from a string of length 127 [-Wstringop-truncation] 241 | std::strncat(storage, tz_offset, sizeof(storage) - timestamp_len - 1); | ~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ While this is essentially a false positive (we never expect the number of bytes in tz_offset to be too large), the compiler can't actually tell that. Shrink the size of tz_offset to a smaller, but still safe size to eliminate this warning. Signed-off-by: Chris Lalancette --- src/timers.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/timers.cc b/src/timers.cc index 1d3ab9a327..af4767dff9 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -190,8 +190,16 @@ std::string LocalDateTimeString() { std::size_t timestamp_len; long int offset_minutes; char tz_offset_sign = '+'; - // Long enough buffers to avoid format-overflow warnings - char tz_offset[128]; + // tz_offset is set in one of three ways: + // * strftime with %z - This either returns empty or the ISO 8601 time. The maximum length an + // ISO 8601 string can be is 7 (e.g. -03:30, plus trailing zero). + // * snprintf with %c%02li:%02li - The maximum length is 41 (one for %c, up to 19 for %02li, + // one for :, up to 19 %02li, plus trailing zero). + // * A fixed string of "-00:00". The maximum length is 7 (-00:00, plus trailing zero). + // + // Thus, the maximum size this needs to be is 41. + char tz_offset[41]; + // Long enough buffer to avoid format-overflow warnings char storage[128]; #if defined(BENCHMARK_OS_WINDOWS) From 39b5a298a7a502f9b8620127030ba318babdcb53 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 12 Apr 2021 12:46:05 +0100 Subject: [PATCH 272/330] Improve BENCHMARK_UNUSED definition (#1111) Take advantage of C++17's `[[maybe_unused]]` if it's available. Also clang supports `__attribute__((unused))` so add that too. --- include/benchmark/benchmark.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 2e714b6eb0..f8b40ae7d1 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -167,6 +167,12 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_HAS_CXX11 #endif +// This _MSC_VER check should detect VS 2017 v15.3 and newer. +#if __cplusplus >= 201703L || \ + (defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L) +#define BENCHMARK_HAS_CXX17 +#endif + #include #include @@ -199,13 +205,19 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); TypeName& operator=(const TypeName&) = delete #endif -#if defined(__GNUC__) +#ifdef BENCHMARK_HAS_CXX17 +#define BENCHMARK_UNUSED [[maybe_unused]] +#elif defined(__GNUC__) || defined(__clang__) #define BENCHMARK_UNUSED __attribute__((unused)) +#else +#define BENCHMARK_UNUSED +#endif + +#if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) #define BENCHMARK_NOEXCEPT noexcept #define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) #elif defined(_MSC_VER) && !defined(__clang__) -#define BENCHMARK_UNUSED #define BENCHMARK_ALWAYS_INLINE __forceinline #if _MSC_VER >= 1900 #define BENCHMARK_NOEXCEPT noexcept @@ -216,7 +228,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #endif #define __func__ __FUNCTION__ #else -#define BENCHMARK_UNUSED #define BENCHMARK_ALWAYS_INLINE #define BENCHMARK_NOEXCEPT #define BENCHMARK_NOEXCEPT_OP(x) From 2dad9aef789462847ce77ac5aabb662e8f75bbf5 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 19 Apr 2021 10:51:12 +0100 Subject: [PATCH 273/330] Re-enable bazel without bazelisk and with scoped build/test targets (#1109) * See if bazel "just works" now (almost) * Add cache and better bazel test command line * Narrow focus of bazel build --- .github/workflows/bazel.yml | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 182e9040de..e2a5b7d857 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -11,23 +11,20 @@ jobs: steps: - uses: actions/checkout@v1 -# - name: mount bazel cache -# uses: actions/cache@v1 -# with: -# path: "/home/runner/.cache/bazel" -# key: bazel + - name: mount bazel cache + uses: actions/cache@v2.0.0 + env: + cache-name: bazel-cache + with: + path: "~/.cache/bazel" + key: ${{ env.cache-name }}-${{ runner.os }}-${{ github.ref }} + restore-keys: | + ${{ env.cache-name }}-${{ runner.os }}-master -# - name: install bazelisk -# run: | -# curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.1.0/bazelisk-linux-amd64" -# mkdir -p "${GITHUB_WORKSPACE}/bin/" -# mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" -# chmod +x "${GITHUB_WORKSPACE}/bin/bazel" - -# - name: build -# run: | -# "${GITHUB_WORKSPACE}/bin/bazel" build //... + - name: build + run: | + bazel build //:benchmark //:benchmark_main //test/... -# - name: test -# run: | -# "${GITHUB_WORKSPACE}/bin/bazel" test //test/... + - name: test + run: | + bazel test --test_output=errors //test/... From 0882a74c8b7866c3a49f01f00a24345d23079282 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 19 Apr 2021 17:25:59 +0100 Subject: [PATCH 274/330] Add bazel status to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 50cb89bf5f..ceedb33436 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Benchmark [![build-and-test](https://github.com/google/benchmark/workflows/build-and-test/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Abuild-and-test) +[![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml) [![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) From 69054ae50e07e9de7cb27f9e2d1d355f74605524 Mon Sep 17 00:00:00 2001 From: Matt Armstrong Date: Mon, 19 Apr 2021 23:16:05 -0700 Subject: [PATCH 275/330] Use fewer ramp up repetitions when KeepRunningBatch is used (#1113) Use the benchmark's reported iteration count when estimating iterations for the next repetition, rather than the requested iteration count. When the benchmark uses KeepRunningBatch the actual iteration count can be larger than the one the runner requested. Prior to this fix the runner was underestimating the next iteration count, sometimes significantly so. Consider the case of a benchmark using a batch size of 1024. Prior to this change, the benchmark runner would attempt iteration counts 1, 10, 100 and 1000, yet the benchmark itself would do the same amount of work each time: a single batch of 1024 iterations. The discrepancy could also contribute to estimation errors once the benchmark time reached 10% of the target. For example, if the very first batch of 1024 iterations reached 10% of benchmark_min_min time, the runner would attempt to scale that to 100% from a basis of one iteration rather than 1024. This bug was particularly noticeable in benchmarks with large batch sizes, especially when the benchmark also had slow set up or tear down phases. With this fix in place it is possible to use KeepRunningBatch to achieve a kind of "minimum iteration count" feature by using a larger fixed batch size. For example, a benchmark may build a map of 500K elements and test a "find" operation. There is no point in running "find" just 1, 10, 100, etc., times. The benchmark can now pick a batch size of something like 10K, and the runner will arrive at the final max iteration count with in noticeably fewer repetitions. --- src/benchmark_runner.cc | 8 +++++--- test/basic_test.cc | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 7bc6b6329e..d081aa869d 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -109,7 +109,7 @@ BenchmarkReporter::Run CreateRunReport( } // Execute one thread of benchmark b for the specified number of iterations. -// Adds the stats collected for the thread into *total. +// Adds the stats collected for the thread into manager->results. void RunInThread(const BenchmarkInstance* b, IterationCount iters, int thread_id, ThreadManager* manager) { internal::ThreadTimer timer( @@ -236,8 +236,10 @@ class BenchmarkRunner { VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; - // So for how long were we running? - i.iters = iters; + // By using KeepRunningBatch a benchmark can iterate more times than + // requested, so take the iteration count from i.results. + i.iters = i.results.iterations / b.threads; + // Base decisions off of real time if requested by this benchmark. i.seconds = i.results.cpu_time_used; if (b.use_manual_time) { diff --git a/test/basic_test.cc b/test/basic_test.cc index 5f3dd1a3ee..33642211e2 100644 --- a/test/basic_test.cc +++ b/test/basic_test.cc @@ -108,15 +108,30 @@ void BM_KeepRunning(benchmark::State& state) { BENCHMARK(BM_KeepRunning); void BM_KeepRunningBatch(benchmark::State& state) { - // Choose a prime batch size to avoid evenly dividing max_iterations. - const benchmark::IterationCount batch_size = 101; + // Choose a batch size >1000 to skip the typical runs with iteration + // targets of 10, 100 and 1000. If these are not actually skipped the + // bug would be detectable as consecutive runs with the same iteration + // count. Below we assert that this does not happen. + const benchmark::IterationCount batch_size = 1009; + + static benchmark::IterationCount prior_iter_count = 0; benchmark::IterationCount iter_count = 0; while (state.KeepRunningBatch(batch_size)) { iter_count += batch_size; } assert(state.iterations() == iter_count); + + // Verify that the iteration count always increases across runs (see + // comment above). + assert(iter_count == batch_size // max_iterations == 1 + || iter_count > prior_iter_count); // max_iterations > batch_size + prior_iter_count = iter_count; } -BENCHMARK(BM_KeepRunningBatch); +// Register with a fixed repetition count to establish the invariant that +// the iteration count should always change across runs. This overrides +// the --benchmark_repetitions command line flag, which would otherwise +// cause this test to fail if set > 1. +BENCHMARK(BM_KeepRunningBatch)->Repetitions(1); void BM_RangedFor(benchmark::State& state) { benchmark::IterationCount iter_count = 0; From c05843a9f622db08ad59804c190f98879b76beba Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Fri, 23 Apr 2021 14:33:22 +0300 Subject: [PATCH 276/330] [sysinfo] Fix CPU Frequency reading on AMD Ryzen CPU's (#1117) Currently, i get: ``` Run on (32 X 7326.56 MHz CPU s) CPU Caches: L1 Data 32 KiB (x16) L1 Instruction 32 KiB (x16) L2 Unified 512 KiB (x16) L3 Unified 32768 KiB (x2) ``` which seems mostly right, except that the frequency is rather bogus. Yes, i guess the CPU could theoretically achieve that, but i have 3.6GHz configured, and scaling disabled. So we clearly read the wrong thing. With this fix, i now get the expected ``` Run on (32 X 3598.53 MHz CPU s) CPU Caches: L1 Data 32 KiB (x16) L1 Instruction 32 KiB (x16) L2 Unified 512 KiB (x16) L3 Unified 32768 KiB (x2) ``` --- include/benchmark/benchmark.h | 2 +- src/sysinfo.cc | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index f8b40ae7d1..44d023fdf2 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1324,9 +1324,9 @@ struct CPUInfo { }; int num_cpus; + Scaling scaling; double cycles_per_second; std::vector caches; - Scaling scaling; std::vector load_avg; static const CPUInfo& Get(); diff --git a/src/sysinfo.cc b/src/sysinfo.cc index b30b4f832e..c1969ea2d3 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -530,7 +530,11 @@ int GetNumCPUs() { BENCHMARK_UNREACHABLE(); } -double GetCPUCyclesPerSecond() { +double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { + // Currently, scaling is only used on linux path here, + // suppress diagnostics about it being unused on other paths. + (void)scaling; + #if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN long freq; @@ -541,8 +545,15 @@ double GetCPUCyclesPerSecond() { // cannot always be relied upon. The same reasons apply to /proc/cpuinfo as // well. if (ReadFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq) - // If CPU scaling is in effect, we want to use the *maximum* frequency, - // not whatever CPU speed some random processor happens to be using now. + // If CPU scaling is disabled, use the the *current* frequency. + // Note that we specifically don't want to read cpuinfo_cur_freq, + // because it is only readable by root. + || (scaling == CPUInfo::Scaling::DISABLED && + ReadFromFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq", + &freq)) + // Otherwise, if CPU scaling may be in effect, we want to use + // the *maximum* frequency, not whatever CPU speed some random processor + // happens to be using now. || ReadFromFile("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", &freq)) { // The value is in kHz (as the file name suggests). For example, on a @@ -701,12 +712,11 @@ const CPUInfo& CPUInfo::Get() { CPUInfo::CPUInfo() : num_cpus(GetNumCPUs()), - cycles_per_second(GetCPUCyclesPerSecond()), - caches(GetCacheSizes()), scaling(CpuScaling(num_cpus)), + cycles_per_second(GetCPUCyclesPerSecond(scaling)), + caches(GetCacheSizes()), load_avg(GetLoadAvg()) {} - const SystemInfo& SystemInfo::Get() { static const SystemInfo* info = new SystemInfo(); return *info; From 362c2ab9c6cc01948a1adbd88fb16d67f2cb880a Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 24 Apr 2021 13:36:38 +0300 Subject: [PATCH 277/330] [tools] Don't forget to print UTest when printing aggregates only This probably regressed in #1042. --- tools/gbench/report.py | 83 ++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index bf29492ed9..1b87df2456 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -301,26 +301,23 @@ def get_color(res): fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}" for benchmark in json_diff_report: # *If* we were asked to only include aggregates, - # and if it is non-aggregate, then skip it. - if include_aggregates_only and 'run_type' in benchmark: - if benchmark['run_type'] != 'aggregate': - continue - - for measurement in benchmark['measurements']: - output_strs += [color_format(use_color, - fmt_str, - BC_HEADER, - benchmark['name'], - first_col_width, - get_color(measurement['time']), - measurement['time'], - get_color(measurement['cpu']), - measurement['cpu'], - measurement['real_time'], - measurement['real_time_other'], - measurement['cpu_time'], - measurement['cpu_time_other'], - endc=BC_ENDC)] + # and if it is non-aggregate, then don't print it. + if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] != 'aggregate': + for measurement in benchmark['measurements']: + output_strs += [color_format(use_color, + fmt_str, + BC_HEADER, + benchmark['name'], + first_col_width, + get_color(measurement['time']), + measurement['time'], + get_color(measurement['cpu']), + measurement['cpu'], + measurement['real_time'], + measurement['real_time_other'], + measurement['cpu_time'], + measurement['cpu_time_other'], + endc=BC_ENDC)] # After processing the measurements, if requested and # if applicable (e.g. u-test exists for given benchmark), @@ -643,6 +640,52 @@ def test_json_diff_report_pretty_printing(self): parts = [x for x in output_lines[i].split(' ') if x] self.assertEqual(expect_lines[i], parts) + def test_json_diff_report_pretty_printing_aggregates_only(self): + expect_lines = [ + ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], + ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], + ['BM_Two_pvalue', + '0.6985', + '0.6985', + 'U', + 'Test,', + 'Repetitions:', + '2', + 'vs', + '2.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], + ['short_pvalue', + '0.7671', + '0.1489', + 'U', + 'Test,', + 'Repetitions:', + '2', + 'vs', + '3.', + 'WARNING:', + 'Results', + 'unreliable!', + '9+', + 'repetitions', + 'recommended.'], + ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], + ] + output_lines_with_header = print_difference_report( + self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False) + output_lines = output_lines_with_header[2:] + print("\n") + print("\n".join(output_lines_with_header)) + self.assertEqual(len(output_lines), len(expect_lines)) + for i in range(0, len(output_lines)): + parts = [x for x in output_lines[i].split(' ') if x] + self.assertEqual(expect_lines[i], parts) + def test_json_diff_report(self): expected_output = [ { From 058fb588b6dc7936927e6a4192a840b8195b59a3 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Sat, 24 Apr 2021 13:43:22 +0300 Subject: [PATCH 278/330] [tools] Fix dumb mistake in previous commit - print aggregates only means aggregates, not non-aggregates --- tools/gbench/report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 1b87df2456..69b643e92b 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -302,7 +302,7 @@ def get_color(res): for benchmark in json_diff_report: # *If* we were asked to only include aggregates, # and if it is non-aggregate, then don't print it. - if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] != 'aggregate': + if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate': for measurement in benchmark['measurements']: output_strs += [color_format(use_color, fmt_str, @@ -642,8 +642,7 @@ def test_json_diff_report_pretty_printing(self): def test_json_diff_report_pretty_printing_aggregates_only(self): expect_lines = [ - ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'], - ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'], + ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'], ['BM_Two_pvalue', '0.6985', '0.6985', @@ -659,6 +658,8 @@ def test_json_diff_report_pretty_printing_aggregates_only(self): '9+', 'repetitions', 'recommended.'], + ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'], + ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'], ['short_pvalue', '0.7671', '0.1489', @@ -674,7 +675,6 @@ def test_json_diff_report_pretty_printing_aggregates_only(self): '9+', 'repetitions', 'recommended.'], - ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'], ] output_lines_with_header = print_difference_report( self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False) From d882be1ea9b1d4413a717202b22b366227bc0b99 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Mon, 26 Apr 2021 04:15:07 -0400 Subject: [PATCH 279/330] fix cmake issue with referencing a non-existing function argument (#1118) according to https://cmake.org/cmake/help/latest/command/function.html, "Referencing to ARGV# arguments beyond ARGC have undefined behavior.", which I hit with cmake 3.19.7. This uses ARGC to check whether ARGV1 has been passed before referencing it. --- cmake/AddCXXCompilerFlag.cmake | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/AddCXXCompilerFlag.cmake b/cmake/AddCXXCompilerFlag.cmake index d0d2099814..858589e977 100644 --- a/cmake/AddCXXCompilerFlag.cmake +++ b/cmake/AddCXXCompilerFlag.cmake @@ -34,9 +34,11 @@ function(add_cxx_compiler_flag FLAG) check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") if(${MANGLED_FLAG}) - set(VARIANT ${ARGV1}) - if(ARGV1) + if(ARGC GREATER 1) + set(VARIANT ${ARGV1}) string(TOUPPER "_${VARIANT}" VARIANT) + else() + set(VARIANT "") endif() set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${BENCHMARK_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) endif() @@ -49,9 +51,11 @@ function(add_required_cxx_compiler_flag FLAG) check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") if(${MANGLED_FLAG}) - set(VARIANT ${ARGV1}) - if(ARGV1) + if(ARGC GREATER 1) + set(VARIANT ${ARGV1}) string(TOUPPER "_${VARIANT}" VARIANT) + else() + set(VARIANT "") endif() set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) From 86da5ec294d09842f93fadb391d91f59709b7269 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 26 Apr 2021 18:26:33 +0100 Subject: [PATCH 280/330] Add verbosity to CI logs (#1122) --- .github/workflows/bazel.yml | 4 ++-- .github/workflows/build-and-test.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index e2a5b7d857..99f7e26054 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -24,7 +24,7 @@ jobs: - name: build run: | bazel build //:benchmark //:benchmark_main //test/... - + - name: test run: | - bazel test --test_output=errors //test/... + bazel test --test_output=all //test/... diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f0f0626d74..4007aa8e44 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -35,4 +35,4 @@ jobs: - name: test shell: bash working-directory: ${{ runner.workspace }}/_build - run: ctest -C ${{ matrix.build_type }} + run: ctest -C ${{ matrix.build_type }} -VV From 264976def327306a4a205857d98ec6792a20cae2 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 27 Apr 2021 08:24:27 +0100 Subject: [PATCH 281/330] Fix windows warning on type conversion (#1121) --- src/benchmark_register.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark_register.h b/src/benchmark_register.h index c774e6f8ea..09496607f2 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -23,7 +23,7 @@ AddPowers(std::vector* dst, T lo, T hi, int mult) { static const T kmax = std::numeric_limits::max(); // Space out the values in multiples of "mult" - for (T i = 1; i <= hi; i *= mult) { + for (T i = static_cast(1); i <= hi; i *= mult) { if (i >= lo) { dst->push_back(i); } From 835951aa44c2f802b4d563d533eac34565848eb0 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 27 Apr 2021 16:54:53 +0100 Subject: [PATCH 282/330] Be compliant and return 0 from main. Fixes #713 --- include/benchmark/benchmark.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 44d023fdf2..881ce9e556 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1301,6 +1301,7 @@ class Fixture : public internal::Benchmark { ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ ::benchmark::RunSpecifiedBenchmarks(); \ + return 0; \ } \ int main(int, char**) From 376ebc26354ca2b79af94467133f3c35b539627e Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Wed, 28 Apr 2021 01:25:29 -0700 Subject: [PATCH 283/330] Support optional, user-directed collection of performance counters (#1114) * Support optional, user-directed collection of performance counters The patch allows an engineer wishing to drill into the root causes of a regression, for example. Currently, only single threaded runs are supported. The feature is a build-time opt in, and then a runtime opt in. The engineer may run the benchmark executable, passing a list of performance counter names (using libpfm's naming scheme) at the command line. The counter values will then be collected and reported back as UserCounters. This is different from #240 in that it is a benchmark user opt-in, and the counter collection is transparent to the benchmark. Currently, this is only supported on platforms where libpfm is supported. libpfm: http://perfmon2.sourceforge.net/ * 'Use' values param in Snapshot when BENCHMARK_OS_WINDOWS This is to avoid unused parameter warning-as-error * Added missing include for in perf_counters.cc * Moved doc to docs * Added license blurbs --- .../workflows/build-and-test-perfcounters.yml | 44 +++++ CMakeLists.txt | 4 + README.md | 2 + cmake/Modules/FindPFM.cmake | 19 ++ docs/perf_counters.md | 35 ++++ include/benchmark/benchmark.h | 9 +- src/CMakeLists.txt | 6 + src/benchmark.cc | 27 ++- src/benchmark_api_internal.cc | 10 +- src/benchmark_api_internal.h | 3 +- src/benchmark_runner.cc | 29 ++- src/benchmark_runner.h | 2 + src/perf_counters.cc | 128 +++++++++++++ src/perf_counters.h | 172 ++++++++++++++++++ src/string_util.cc | 12 ++ src/string_util.h | 2 + test/CMakeLists.txt | 4 + test/perf_counters_gtest.cc | 95 ++++++++++ test/perf_counters_test.cc | 27 +++ test/string_util_gtest.cc | 8 + 20 files changed, 621 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/build-and-test-perfcounters.yml create mode 100644 cmake/Modules/FindPFM.cmake create mode 100644 docs/perf_counters.md create mode 100644 src/perf_counters.cc create mode 100644 src/perf_counters.h create mode 100644 test/perf_counters_gtest.cc create mode 100644 test/perf_counters_test.cc diff --git a/.github/workflows/build-and-test-perfcounters.yml b/.github/workflows/build-and-test-perfcounters.yml new file mode 100644 index 0000000000..dfb88cbc3e --- /dev/null +++ b/.github/workflows/build-and-test-perfcounters.yml @@ -0,0 +1,44 @@ +name: build-and-test-perfcounters + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + job: + # TODO(dominic): Extend this to include compiler and set through env: CC/CXX. + name: ${{ matrix.os }}.${{ matrix.build_type }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-16.04, ubuntu-20.04] + build_type: ['Release', 'Debug'] + steps: + - uses: actions/checkout@v2 + + - name: install libpfm + run: sudo apt install libpfm4-dev + + - name: create build environment + run: cmake -E make_directory ${{ runner.workspace }}/_build + + - name: configure cmake + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake -DBENCHMARK_ENABLE_LIBPFM=1 -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + + - name: build + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} + + # Skip testing, for now. It seems perf_event_open does not succeed on the + # hosting machine, very likely a permissions issue. + # TODO(mtrofin): Enable test. + # - name: test + # shell: bash + # working-directory: ${{ runner.workspace }}/_build + # run: sudo ctest -C ${{ matrix.build_type }} --rerun-failed --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 1007254520..7e0f251fe5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -270,6 +270,10 @@ cxx_feature_check(STEADY_CLOCK) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +if (BENCHMARK_ENABLE_LIBPFM) + find_package(PFM) +endif() + # Set up directories include_directories(${PROJECT_SOURCE_DIR}/include) diff --git a/README.md b/README.md index ceedb33436..e8c65f8d87 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,8 @@ too (`-lkstat`). [Setting the Time Unit](#setting-the-time-unit) +[User-Requested Performance Counters](docs/perf_counters.md) + [Preventing Optimization](#preventing-optimization) [Reporting Statistics](#reporting-statistics) diff --git a/cmake/Modules/FindPFM.cmake b/cmake/Modules/FindPFM.cmake new file mode 100644 index 0000000000..553d458c00 --- /dev/null +++ b/cmake/Modules/FindPFM.cmake @@ -0,0 +1,19 @@ +# If successful, the following variables will be defined: +# HAVE_LIBPFM. +# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence. +include(CheckIncludeFile) +include(CheckLibraryExists) +enable_language(C) + +check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE) +if(HAVE_LIBPFM_INITIALIZE) + check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H) + check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H) + check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H) + if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H) + message("Using Perf Counters.") + set(HAVE_LIBPFM 1) + endif() +else() + message("Perf Counters support requested, but was unable to find libpfm.") +endif() diff --git a/docs/perf_counters.md b/docs/perf_counters.md new file mode 100644 index 0000000000..43ff451791 --- /dev/null +++ b/docs/perf_counters.md @@ -0,0 +1,35 @@ + + +# User-Requested Performance Counters + +When running benchmarks, the user may choose to request collection of +performance counters. This may be useful in investigation scenarios - narrowing +down the cause of a regression; or verifying that the underlying cause of a +performance improvement matches expectations. + +This feature is available if: + +* The benchmark is run on an architecture featuring a Performance Monitoring + Unit (PMU), +* The benchmark is compiled with support for collecting counters. Currently, + this requires [libpfm](http://perfmon2.sourceforge.net/) be available at build + time, and +* Currently, there is a limitation that the benchmark be run on one thread. + +The feature does not require modifying benchmark code. Counter collection is +handled at the boundaries where timer collection is also handled. + +To opt-in: + +* Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. +* Enable the cmake flag BENCHMARK_ENABLE_LIBPFM. + +To use, pass a comma-separated list of counter names through the +`--benchmark_perf_counters` flag. The names are decoded through libpfm - meaning, +they are platform specific, but some (e.g. `CYCLES` or `INSTRUCTIONS`) are +mapped by libpfm to platform-specifics - see libpfm +[documentation](http://perfmon2.sourceforge.net/docs.html) for more details. + +The counter values are reported back through the [User Counters](../README.md#custom-counters) +mechanism, meaning, they are available in all the formats (e.g. JSON) supported +by User Counters. \ No newline at end of file diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 881ce9e556..664422b347 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -448,6 +448,7 @@ struct Statistics { struct BenchmarkInstance; class ThreadTimer; class ThreadManager; +class PerfCountersMeasurement; enum AggregationReportMode #if defined(BENCHMARK_HAS_CXX11) @@ -687,15 +688,17 @@ class State { private: State(IterationCount max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager); + internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. bool KeepRunningInternal(IterationCount n, bool is_batch); void FinishKeepRunning(); - internal::ThreadTimer* timer_; - internal::ThreadManager* manager_; + internal::ThreadTimer* const timer_; + internal::ThreadManager* const manager_; + internal::PerfCountersMeasurement* const perf_counters_measurement_; friend struct internal::BenchmarkInstance; }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35d559eeae..a6c8e9a7a0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,6 +28,12 @@ target_include_directories(benchmark PUBLIC $ ) +# libpfm, if available +if (HAVE_LIBPFM) + target_link_libraries(benchmark libpfm.a) + add_definitions(-DHAVE_LIBPFM) +endif() + # Link threads. target_link_libraries(benchmark ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) find_library(LIBRT rt) diff --git a/src/benchmark.cc b/src/benchmark.cc index ffe4bf45a6..1fea654c42 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -45,6 +45,7 @@ #include "internal_macros.h" #include "log.h" #include "mutex.h" +#include "perf_counters.h" #include "re.h" #include "statistics.h" #include "string_util.h" @@ -106,6 +107,10 @@ DEFINE_bool(benchmark_counters_tabular, false); // The level of verbose logging to output DEFINE_int32(v, 0); +// List of additional perf counters to collect, in libpfm format. For more +// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html +DEFINE_string(benchmark_perf_counters, ""); + namespace benchmark { namespace internal { @@ -117,7 +122,8 @@ void UseCharPointer(char const volatile*) {} State::State(IterationCount max_iters, const std::vector& ranges, int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager) + internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), @@ -130,7 +136,8 @@ State::State(IterationCount max_iters, const std::vector& ranges, thread_index(thread_i), threads(n_threads), timer_(timer), - manager_(manager) { + manager_(manager), + perf_counters_measurement_(perf_counters_measurement) { CHECK(max_iterations != 0) << "At least one iteration must be run"; CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; @@ -163,11 +170,23 @@ void State::PauseTiming() { // Add in time accumulated so far CHECK(started_ && !finished_ && !error_occurred_); timer_->StopTimer(); + if (perf_counters_measurement_) { + auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); + for (const auto& name_and_measurement : measurements) { + auto name = name_and_measurement.first; + auto measurement = name_and_measurement.second; + CHECK_EQ(counters[name], 0.0); + counters[name] = Counter(measurement, Counter::kAvgIterations); + } + } } void State::ResumeTiming() { CHECK(started_ && !finished_ && !error_occurred_); timer_->StartTimer(); + if (perf_counters_measurement_) { + perf_counters_measurement_->Start(); + } } void State::SkipWithError(const char* msg) { @@ -457,7 +476,9 @@ void ParseCommandLineFlags(int* argc, char** argv) { ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || ParseBoolFlag(argv[i], "benchmark_counters_tabular", &FLAGS_benchmark_counters_tabular) || - ParseInt32Flag(argv[i], "v", &FLAGS_v)) { + ParseInt32Flag(argv[i], "v", &FLAGS_v) || + ParseStringFlag(argv[i], "benchmark_perf_counters", + &FLAGS_benchmark_perf_counters)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; --(*argc); diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index d468a257e3..804ef894ab 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -3,10 +3,12 @@ namespace benchmark { namespace internal { -State BenchmarkInstance::Run(IterationCount iters, int thread_id, - internal::ThreadTimer* timer, - internal::ThreadManager* manager) const { - State st(iters, arg, thread_id, threads, timer, manager); +State BenchmarkInstance::Run( + IterationCount iters, int thread_id, internal::ThreadTimer* timer, + internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement) const { + State st(iters, arg, thread_id, threads, timer, manager, + perf_counters_measurement); benchmark->Run(st); return st; } diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 264eff95c5..b740bce115 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -36,7 +36,8 @@ struct BenchmarkInstance { int threads; // Number of concurrent threads to us State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, - internal::ThreadManager* manager) const; + internal::ThreadManager* manager, + internal::PerfCountersMeasurement* perf_counters_measurement) const; }; bool FindBenchmarksInternal(const std::string& re, diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index d081aa869d..083d18492a 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -45,6 +45,7 @@ #include "internal_macros.h" #include "log.h" #include "mutex.h" +#include "perf_counters.h" #include "re.h" #include "statistics.h" #include "string_util.h" @@ -111,12 +112,14 @@ BenchmarkReporter::Run CreateRunReport( // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into manager->results. void RunInThread(const BenchmarkInstance* b, IterationCount iters, - int thread_id, ThreadManager* manager) { + int thread_id, ThreadManager* manager, + PerfCountersMeasurement* perf_counters_measurement) { internal::ThreadTimer timer( b->measure_process_cpu_time ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); - State st = b->Run(iters, thread_id, &timer, manager); + State st = + b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { @@ -143,7 +146,12 @@ class BenchmarkRunner { : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations != 0), pool(b.threads - 1), - iters(has_explicit_iteration_count ? b.iterations : 1) { + iters(has_explicit_iteration_count ? b.iterations : 1), + perf_counters_measurement( + PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), + perf_counters_measurement_ptr(perf_counters_measurement.IsValid() + ? &perf_counters_measurement + : nullptr) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -155,6 +163,11 @@ class BenchmarkRunner { internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = (b.aggregation_report_mode & internal::ARM_FileReportAggregatesOnly); + CHECK(b.threads == 1 || !perf_counters_measurement.IsValid()) + << "Perf counters are not supported in multi-threaded cases.\n"; + CHECK(FLAGS_benchmark_perf_counters.empty() || + perf_counters_measurement.IsValid()) + << "Perf counters were requested but could not be set up."; } for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { @@ -192,6 +205,9 @@ class BenchmarkRunner { // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. + PerfCountersMeasurement perf_counters_measurement; + PerfCountersMeasurement* const perf_counters_measurement_ptr; + struct IterationResults { internal::ThreadManager::Result results; IterationCount iters; @@ -206,12 +222,12 @@ class BenchmarkRunner { // Run all but one thread in separate threads for (std::size_t ti = 0; ti < pool.size(); ++ti) { pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), - manager.get()); + manager.get(), perf_counters_measurement_ptr); } // And run one thread here directly. // (If we were asked to run just one thread, we don't create new threads.) // Yes, we need to do this here *after* we start the separate threads. - RunInThread(&b, iters, 0, manager.get()); + RunInThread(&b, iters, 0, manager.get(), perf_counters_measurement_ptr); // The main thread has finished. Now let's wait for the other threads. manager->WaitForAllThreads(); @@ -331,7 +347,8 @@ class BenchmarkRunner { memory_manager->Start(); std::unique_ptr manager; manager.reset(new internal::ThreadManager(1)); - RunInThread(&b, memory_iterations, 0, manager.get()); + RunInThread(&b, memory_iterations, 0, manager.get(), + perf_counters_measurement_ptr); manager->WaitForAllThreads(); manager.reset(); diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 96e8282a11..9b0cf2a64e 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -26,6 +26,8 @@ DECLARE_bool(benchmark_report_aggregates_only); DECLARE_bool(benchmark_display_aggregates_only); +DECLARE_string(benchmark_perf_counters); + namespace benchmark { namespace internal { diff --git a/src/perf_counters.cc b/src/perf_counters.cc new file mode 100644 index 0000000000..eb28cd99f8 --- /dev/null +++ b/src/perf_counters.cc @@ -0,0 +1,128 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "perf_counters.h" + +#include +#include + +#if defined HAVE_LIBPFM +#include "perfmon/pfmlib.h" +#include "perfmon/pfmlib_perf_event.h" +#endif + +namespace benchmark { +namespace internal { + +constexpr size_t PerfCounterValues::kMaxCounters; + +#if defined HAVE_LIBPFM +const bool PerfCounters::kSupported = true; + +bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; } + +PerfCounters PerfCounters::Create( + const std::vector& counter_names) { + if (counter_names.empty()) { + return NoCounters(); + } + if (counter_names.size() > PerfCounterValues::kMaxCounters) { + GetErrorLogInstance() + << counter_names.size() + << " counters were requested. The minimum is 1, the maximum is " + << PerfCounterValues::kMaxCounters << "\n"; + return NoCounters(); + } + std::vector counter_ids(counter_names.size()); + + const int mode = PFM_PLM3; // user mode only + for (size_t i = 0; i < counter_names.size(); ++i) { + const bool is_first = i == 0; + struct perf_event_attr attr{}; + attr.size = sizeof(attr); + const int group_id = !is_first ? counter_ids[0] : -1; + const auto& name = counter_names[i]; + if (name.empty()) { + GetErrorLogInstance() << "A counter name was the empty string\n"; + return NoCounters(); + } + pfm_perf_encode_arg_t arg{}; + arg.attr = &attr; + + const int pfm_get = + pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT, &arg); + if (pfm_get != PFM_SUCCESS) { + GetErrorLogInstance() << "Unknown counter name: " << name << "\n"; + return NoCounters(); + } + attr.disabled = is_first; + attr.pinned = is_first; + attr.exclude_kernel = true; + attr.exclude_user = false; + attr.exclude_hv = true; + // Read all counters in one read. + attr.read_format = PERF_FORMAT_GROUP; + + int id = -1; + static constexpr size_t kNrOfSyscallRetries = 5; + // Retry syscall as it was interrupted often (b/64774091). + for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; + ++num_retries) { + id = perf_event_open(&attr, 0, -1, group_id, 0); + if (id >= 0 || errno != EINTR) { + break; + } + } + if (id < 0) { + GetErrorLogInstance() + << "Failed to get a file descriptor for " << name << "\n"; + return NoCounters(); + } + + counter_ids[i] = id; + } + if (ioctl(counter_ids[0], PERF_EVENT_IOC_ENABLE) != 0) { + GetErrorLogInstance() << "Failed to start counters\n"; + return NoCounters(); + } + + return PerfCounters(counter_names, std::move(counter_ids)); +} + +PerfCounters::~PerfCounters() { + if (counter_ids_.empty()) { + return; + } + ioctl(counter_ids_[0], PERF_EVENT_IOC_DISABLE); + for (int fd : counter_ids_) { + close(fd); + } +} +#else // defined HAVE_LIBPFM +const bool PerfCounters::kSupported = false; + +bool PerfCounters::Initialize() { return false; } + +PerfCounters PerfCounters::Create( + const std::vector& counter_names) { + if (!counter_names.empty()) { + GetErrorLogInstance() << "Performance counters not supported."; + } + return NoCounters(); +} + +PerfCounters::~PerfCounters() = default; +#endif // defined HAVE_LIBPFM +} // namespace internal +} // namespace benchmark diff --git a/src/perf_counters.h b/src/perf_counters.h new file mode 100644 index 0000000000..0c2c4616b3 --- /dev/null +++ b/src/perf_counters.h @@ -0,0 +1,172 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_PERF_COUNTERS_H +#define BENCHMARK_PERF_COUNTERS_H + +#include +#include +#include + +#include "benchmark/benchmark.h" +#include "check.h" +#include "log.h" + +#ifndef BENCHMARK_OS_WINDOWS +#include +#endif + +namespace benchmark { +namespace internal { + +// Typically, we can only read a small number of counters. There is also a +// padding preceding counter values, when reading multiple counters with one +// syscall (which is desirable). PerfCounterValues abstracts these details. +// The implementation ensures the storage is inlined, and allows 0-based +// indexing into the counter values. +// The object is used in conjunction with a PerfCounters object, by passing it +// to Snapshot(). The values are populated such that +// perfCounters->names()[i]'s value is obtained at position i (as given by +// operator[]) of this object. +class PerfCounterValues { + public: + explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { + CHECK_LE(nr_counters_, kMaxCounters); + } + + uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } + + static constexpr size_t kMaxCounters = 3; + + private: + friend class PerfCounters; + // Get the byte buffer in which perf counters can be captured. + // This is used by PerfCounters::Read + std::pair get_data_buffer() { + return {reinterpret_cast(values_.data()), + sizeof(uint64_t) * (kPadding + nr_counters_)}; + } + + static constexpr size_t kPadding = 1; + std::array values_; + const size_t nr_counters_; +}; + +// Collect PMU counters. The object, once constructed, is ready to be used by +// calling read(). PMU counter collection is enabled from the time create() is +// called, to obtain the object, until the object's destructor is called. +class PerfCounters final { + public: + // True iff this platform supports performance counters. + static const bool kSupported; + + bool IsValid() const { return is_valid_; } + static PerfCounters NoCounters() { return PerfCounters(); } + + ~PerfCounters(); + PerfCounters(PerfCounters&&) = default; + PerfCounters(const PerfCounters&) = delete; + + // Platform-specific implementations may choose to do some library + // initialization here. + static bool Initialize(); + + // Return a PerfCounters object ready to read the counters with the names + // specified. The values are user-mode only. The counter name format is + // implementation and OS specific. + // TODO: once we move to C++-17, this should be a std::optional, and then the + // IsValid() boolean can be dropped. + static PerfCounters Create(const std::vector& counter_names); + + // Take a snapshot of the current value of the counters into the provided + // valid PerfCounterValues storage. The values are populated such that: + // names()[i]'s value is (*values)[i] + BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) { +#ifndef BENCHMARK_OS_WINDOWS + assert(values != nullptr); + assert(IsValid()); + auto buffer = values->get_data_buffer(); + auto read_bytes = ::read(counter_ids_[0], buffer.first, buffer.second); + return static_cast(read_bytes) == buffer.second; +#else + (void)values; + return false; +#endif + } + + const std::vector& names() const { return counter_names_; } + size_t num_counters() const { return counter_names_.size(); } + + private: + PerfCounters(const std::vector& counter_names, + std::vector&& counter_ids) + : counter_ids_(std::move(counter_ids)), + counter_names_(counter_names), + is_valid_(true) {} + PerfCounters() : is_valid_(false) {} + + std::vector counter_ids_; + const std::vector counter_names_; + const bool is_valid_; +}; + +// Typical usage of the above primitives. +class PerfCountersMeasurement final { + public: + PerfCountersMeasurement(PerfCounters&& c) + : counters_(std::move(c)), + start_values_(counters_.IsValid() ? counters_.names().size() : 0), + end_values_(counters_.IsValid() ? counters_.names().size() : 0) {} + + bool IsValid() const { return counters_.IsValid(); } + + BENCHMARK_ALWAYS_INLINE void Start() { + assert(IsValid()); + // Tell the compiler to not move instructions above/below where we take + // the snapshot. + ClobberMemory(); + counters_.Snapshot(&start_values_); + ClobberMemory(); + } + + BENCHMARK_ALWAYS_INLINE std::vector> + StopAndGetMeasurements() { + assert(IsValid()); + // Tell the compiler to not move instructions above/below where we take + // the snapshot. + ClobberMemory(); + counters_.Snapshot(&end_values_); + ClobberMemory(); + + std::vector> ret; + for (size_t i = 0; i < counters_.names().size(); ++i) { + double measurement = static_cast(end_values_[i]) - + static_cast(start_values_[i]); + ret.push_back({counters_.names()[i], measurement}); + } + return ret; + } + + private: + PerfCounters counters_; + PerfCounterValues start_values_; + PerfCounterValues end_values_; +}; + +BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize(); + +} // namespace internal +} // namespace benchmark + +#endif // BENCHMARK_PERF_COUNTERS_H diff --git a/src/string_util.cc b/src/string_util.cc index ac60b5588f..53b1532b94 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -163,6 +163,18 @@ std::string StrFormat(const char* format, ...) { return tmp; } +std::vector StrSplit(const std::string& str, char delim) { + std::vector ret; + size_t first = 0; + size_t next = str.find(delim); + for (; next != std::string::npos; + first = next + 1, next = str.find(first, delim)) { + ret.push_back(str.substr(first, next - first)); + } + ret.push_back(str.substr(first)); + return ret; +} + #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/src/string_util.h b/src/string_util.h index 09d7b4bd2a..6bc28b6912 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -37,6 +37,8 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } +std::vector StrSplit(const std::string& str, char delim); + #ifdef BENCHMARK_STL_ANDROID_GNUSTL /* * GNU STL in Android NDK lacks support for some C++11 functions, including diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c1a3a3fc19..1e7b6829f2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -128,6 +128,9 @@ add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_ compile_output_test(user_counters_test) add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) +compile_output_test(perf_counters_test) +add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) + compile_output_test(internal_threading_test) add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) @@ -196,6 +199,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) + add_gtest(perf_counters_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/test/perf_counters_gtest.cc b/test/perf_counters_gtest.cc new file mode 100644 index 0000000000..47894af4f2 --- /dev/null +++ b/test/perf_counters_gtest.cc @@ -0,0 +1,95 @@ +#include "../src/perf_counters.h" +#include "gtest/gtest.h" + +#ifndef GTEST_SKIP +struct MsgHandler { + void operator=(std::ostream&){} +}; +#define GTEST_SKIP() return MsgHandler() = std::cout +#endif + +using benchmark::internal::PerfCounters; +using benchmark::internal::PerfCounterValues; + +namespace { +const char kGenericPerfEvent1[] = "CYCLES"; +const char kGenericPerfEvent2[] = "BRANCHES"; +const char kGenericPerfEvent3[] = "INSTRUCTIONS"; + +TEST(PerfCountersTest, Init) { + EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); +} + +TEST(PerfCountersTest, OneCounter) { + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Performance counters not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1}).IsValid()); +} + +TEST(PerfCountersTest, NegativeTest) { + if (!PerfCounters::kSupported) { + EXPECT_FALSE(PerfCounters::Initialize()); + return; + } + EXPECT_TRUE(PerfCounters::Initialize()); + EXPECT_FALSE(PerfCounters::Create({}).IsValid()); + EXPECT_FALSE(PerfCounters::Create({""}).IsValid()); + EXPECT_FALSE(PerfCounters::Create({"not a counter name"}).IsValid()); + { + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3}) + .IsValid()); + } + EXPECT_FALSE( + PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}) + .IsValid()); + EXPECT_FALSE(PerfCounters::Create({kGenericPerfEvent3, "not a counter name", + kGenericPerfEvent1}) + .IsValid()); + { + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3}) + .IsValid()); + } + EXPECT_FALSE( + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3, "MISPREDICTED_BRANCH_RETIRED"}) + .IsValid()); +} + +TEST(PerfCountersTest, Read1Counter) { + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + auto counters = PerfCounters::Create({kGenericPerfEvent1}); + EXPECT_TRUE(counters.IsValid()); + PerfCounterValues values1(1); + EXPECT_TRUE(counters.Snapshot(&values1)); + EXPECT_GT(values1[0], 0); + PerfCounterValues values2(1); + EXPECT_TRUE(counters.Snapshot(&values2)); + EXPECT_GT(values2[0], 0); + EXPECT_GT(values2[0], values1[0]); +} + +TEST(PerfCountersTest, Read2Counters) { + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + auto counters = + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); + EXPECT_TRUE(counters.IsValid()); + PerfCounterValues values1(2); + EXPECT_TRUE(counters.Snapshot(&values1)); + EXPECT_GT(values1[0], 0); + EXPECT_GT(values1[1], 0); + PerfCounterValues values2(2); + EXPECT_TRUE(counters.Snapshot(&values2)); + EXPECT_GT(values2[0], 0); + EXPECT_GT(values2[1], 0); +} +} // namespace diff --git a/test/perf_counters_test.cc b/test/perf_counters_test.cc new file mode 100644 index 0000000000..d6e0284d4d --- /dev/null +++ b/test/perf_counters_test.cc @@ -0,0 +1,27 @@ +#undef NDEBUG + +#include "../src/perf_counters.h" + +#include "benchmark/benchmark.h" +#include "output_test.h" + +void BM_Simple(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(state.iterations()); + } +} +BENCHMARK(BM_Simple); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); + +void CheckSimple(Results const& e) { + CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); + CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); +} +CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); + +int main(int argc, char* argv[]) { + if (!benchmark::internal::PerfCounters::kSupported) { + return 0; + } + RunOutputTests(argc, argv); +} diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 01bf155d8c..77a719a69d 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -150,4 +150,12 @@ TEST(StringUtilTest, stod) { #endif } +TEST(StringUtilTest, StrSplit) { + EXPECT_EQ(benchmark::StrSplit("", ','), std::vector{""}); + EXPECT_EQ(benchmark::StrSplit("hello", ','), + std::vector({"hello"})); + EXPECT_EQ(benchmark::StrSplit("hello,there", ','), + std::vector({"hello", "there"})); +} + } // end namespace From 64cb55e91067860548cb95e012a38f2e5b71e026 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 28 Apr 2021 16:44:33 +0100 Subject: [PATCH 284/330] enable markdown rendering on github pages --- _config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index 18854876c6..1fa5ff852b 100644 --- a/_config.yml +++ b/_config.yml @@ -1 +1,2 @@ -theme: jekyll-theme-midnight \ No newline at end of file +theme: jekyll-theme-midnight +markdown: GFM From ba9a763def4eca056d03b1ece2946b2d4ef6dfcb Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 30 Apr 2021 14:22:15 +0100 Subject: [PATCH 285/330] Add multiple compiler support to build-and-test workflow (#1128) * Add 32-bit build support to build-and-test * attempt different yaml multiline string format * syntax fixes to yaml * switch to getting alternative compilers working * remove done TODO * trying to separate out windows * oops, typo. * add TODOs for missing builds wrt travis --- .github/workflows/build-and-test.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4007aa8e44..731a19a364 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -7,15 +7,27 @@ on: branches: [ master ] jobs: + # TODO: add job using container ubuntu:14.04 with and without g++-6 + # TODO: add 32-bit builds (g++ and clang++) for ubuntu (requires g++-multilib and libc6:i386) + # TODO: add coverage build (requires lcov) + # TODO: add clang + libc++ builds for ubuntu + # TODO: add clang + ubsan/asan/msan + libc++ builds for ubuntu job: - # TODO(dominic): Extend this to include compiler and set through env: CC/CXX. - name: ${{ matrix.os }}.${{ matrix.build_type }} + name: ${{ matrix.os }}.${{ matrix.build_type }}.${{ matrix.compiler }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-16.04, ubuntu-20.04, macos-latest, windows-latest] + os: [ubuntu-latest, ubuntu-16.04, ubuntu-20.04, macos-latest] build_type: ['Release', 'Debug'] + compiler: [g++, clang++] + include: + - displayTargetName: windows-latest-release + os: windows-latest + build_type: 'Release' + - displayTargetName: windows-latest-debug + os: windows-latest + build_type: 'Debug' steps: - uses: actions/checkout@v2 @@ -23,9 +35,14 @@ jobs: run: cmake -E make_directory ${{ runner.workspace }}/_build - name: configure cmake + env: + CXX: ${{ matrix.compiler }} shell: bash working-directory: ${{ runner.workspace }}/_build - run: cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + run: > + cmake $GITHUB_WORKSPACE + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - name: build shell: bash From 33c133a2067316e462ad684e8a36207610da1bb6 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 4 May 2021 14:36:11 +0100 Subject: [PATCH 286/330] Add `benchmark_context` flag that allows per-run custom context. (#1127) * Add `benchmark_context` flag that allows per-run custom context. Add support for key-value flags in general. Added test for key-value flags. Added `benchmark_context` flag. Output content of `benchmark_context` to base reporter. Solves the first part of #525. * Docs and better help --- README.md | 21 +++++++++++ src/benchmark.cc | 12 +++++-- src/commandlineflags.cc | 58 ++++++++++++++++++++++++++++++ src/commandlineflags.h | 65 ++++++++++++++++++++-------------- src/reporter.cc | 7 ++++ test/commandlineflags_gtest.cc | 33 +++++++++++++++-- 6 files changed, 165 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e8c65f8d87..478ac89675 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,8 @@ too (`-lkstat`). [Result Comparison](#result-comparison) +[Extra Context](#extra-context) + ### Library [Runtime and Reporting Considerations](#runtime-and-reporting-considerations) @@ -442,6 +444,25 @@ BM_memcpy/32k 1834 ns 1837 ns 357143 It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md) + + +### Extra Context + +Sometimes it's useful to add extra context to the content printed before the +results. By default this section includes information about the CPU on which +the benchmarks are running. If you do want to add more context, you can use +the `benchmark_context` command line flag: + +```bash +$ ./run_benchmarks --benchmark_context=pwd=`pwd` +Run on (1 x 2300 MHz CPU) +pwd: /home/user/benchmark/ +Benchmark Time CPU Iterations +---------------------------------------------------- +BM_memcpy/32 11 ns 11 ns 79545455 +BM_memcpy/32k 2181 ns 2185 ns 324074 +``` + ### Runtime and Reporting Considerations diff --git a/src/benchmark.cc b/src/benchmark.cc index 1fea654c42..930a75671a 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark/benchmark.h" + #include "benchmark_api_internal.h" #include "benchmark_runner.h" #include "internal_macros.h" @@ -104,6 +105,10 @@ DEFINE_string(benchmark_color, "auto"); // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. DEFINE_bool(benchmark_counters_tabular, false); +// Extra context to include in the output formatted as comma-separated key-value +// pairs. +DEFINE_kvpairs(benchmark_context, {}); + // The level of verbose logging to output DEFINE_int32(v, 0); @@ -446,6 +451,7 @@ void PrintUsageAndExit() { " [--benchmark_out_format=]\n" " [--benchmark_color={auto|true|false}]\n" " [--benchmark_counters_tabular={true|false}]\n" + " [--benchmark_context==,...]\n" " [--v=]\n"); exit(0); } @@ -476,9 +482,11 @@ void ParseCommandLineFlags(int* argc, char** argv) { ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || ParseBoolFlag(argv[i], "benchmark_counters_tabular", &FLAGS_benchmark_counters_tabular) || - ParseInt32Flag(argv[i], "v", &FLAGS_v) || ParseStringFlag(argv[i], "benchmark_perf_counters", - &FLAGS_benchmark_perf_counters)) { + &FLAGS_benchmark_perf_counters) || + ParseKeyValueFlag(argv[i], "benchmark_context", + &FLAGS_benchmark_context) || + ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; --(*argc); diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc index 0648fe3a06..5724aaa294 100644 --- a/src/commandlineflags.cc +++ b/src/commandlineflags.cc @@ -20,6 +20,10 @@ #include #include #include +#include +#include + +#include "../src/string_util.h" namespace benchmark { namespace { @@ -78,6 +82,30 @@ bool ParseDouble(const std::string& src_text, const char* str, double* value) { return true; } +// Parses 'str' into KV pairs. If successful, writes the result to *value and +// returns true; otherwise leaves *value unchanged and returns false. +bool ParseKvPairs(const std::string& src_text, const char* str, + std::map* value) { + std::map kvs; + for (const auto& kvpair : StrSplit(str, ',')) { + const auto kv = StrSplit(kvpair, '='); + if (kv.size() != 2) { + std::cerr << src_text << " is expected to be a comma-separated list of " + << "= strings, but actually has value \"" << str + << "\".\n"; + return false; + } + if (!kvs.emplace(kv[0], kv[1]).second) { + std::cerr << src_text << " is expected to contain unique keys but key \"" + << kv[0] << "\" was repeated.\n"; + return false; + } + } + + *value = kvs; + return true; +} + // Returns the name of the environment variable corresponding to the // given flag. For example, FlagToEnvVar("foo") will return // "BENCHMARK_FOO" in the open-source version. @@ -129,6 +157,20 @@ const char* StringFromEnv(const char* flag, const char* default_val) { return value == nullptr ? default_val : value; } +std::map KvPairsFromEnv( + const char* flag, std::map default_val) { + const std::string env_var = FlagToEnvVar(flag); + const char* const value_str = getenv(env_var.c_str()); + + if (value_str == nullptr) return default_val; + + std::map value; + if (!ParseKvPairs("Environment variable " + env_var, value_str, &value)) { + return default_val; + } + return value; +} + // Parses a string as a command line flag. The string should have // the format "--flag=value". When def_optional is true, the "=value" // part can be omitted. @@ -206,6 +248,22 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) { return true; } +bool ParseKeyValueFlag( + const char* str, const char* flag, + std::map* value) { + const char* const value_str = ParseFlagValue(str, flag, false); + + if (value_str == nullptr) return false; + + for (const auto& kvpair : StrSplit(value_str, ',')) { + const auto kv = StrSplit(kvpair, '='); + if (kv.size() != 2) return false; + value->emplace(kv[0], kv[1]); + } + + return true; +} + bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 3a1f6a8dbc..0c988cccb3 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -2,6 +2,7 @@ #define BENCHMARK_COMMANDLINEFLAGS_H_ #include +#include #include // Macro for referencing flags. @@ -12,51 +13,59 @@ #define DECLARE_int32(name) extern int32_t FLAG(name) #define DECLARE_double(name) extern double FLAG(name) #define DECLARE_string(name) extern std::string FLAG(name) +#define DECLARE_kvpairs(name) \ + extern std::map FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val) \ - bool FLAG(name) = \ - benchmark::BoolFromEnv(#name, default_val) -#define DEFINE_int32(name, default_val) \ - int32_t FLAG(name) = \ - benchmark::Int32FromEnv(#name, default_val) -#define DEFINE_double(name, default_val) \ - double FLAG(name) = \ - benchmark::DoubleFromEnv(#name, default_val) -#define DEFINE_string(name, default_val) \ - std::string FLAG(name) = \ - benchmark::StringFromEnv(#name, default_val) +#define DEFINE_bool(name, default_val) \ + bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) +#define DEFINE_int32(name, default_val) \ + int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) +#define DEFINE_double(name, default_val) \ + double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) +#define DEFINE_string(name, default_val) \ + std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) +#define DEFINE_kvpairs(name, default_val) \ + std::map FLAG(name) = \ + benchmark::KvPairsFromEnv(#name, default_val) namespace benchmark { -// Parses a bool from the environment variable -// corresponding to the given flag. +// Parses a bool from the environment variable corresponding to the given flag. // // If the variable exists, returns IsTruthyFlagValue() value; if not, // returns the given default value. bool BoolFromEnv(const char* flag, bool default_val); -// Parses an Int32 from the environment variable -// corresponding to the given flag. +// Parses an Int32 from the environment variable corresponding to the given +// flag. // // If the variable exists, returns ParseInt32() value; if not, returns // the given default value. int32_t Int32FromEnv(const char* flag, int32_t default_val); -// Parses an Double from the environment variable -// corresponding to the given flag. +// Parses an Double from the environment variable corresponding to the given +// flag. // // If the variable exists, returns ParseDouble(); if not, returns // the given default value. double DoubleFromEnv(const char* flag, double default_val); -// Parses a string from the environment variable -// corresponding to the given flag. +// Parses a string from the environment variable corresponding to the given +// flag. // // If variable exists, returns its value; if not, returns // the given default value. const char* StringFromEnv(const char* flag, const char* default_val); +// Parses a set of kvpairs from the environment variable corresponding to the +// given flag. +// +// If variable exists, returns its value; if not, returns +// the given default value. +std::map KvPairsFromEnv( + const char* flag, std::map default_val); + // Parses a string for a bool flag, in the form of either // "--flag=value" or "--flag". // @@ -68,27 +77,31 @@ const char* StringFromEnv(const char* flag, const char* default_val); // true. On failure, returns false without changing *value. bool ParseBoolFlag(const char* str, const char* flag, bool* value); -// Parses a string for an Int32 flag, in the form of -// "--flag=value". +// Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); -// Parses a string for a Double flag, in the form of -// "--flag=value". +// Parses a string for a Double flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseDoubleFlag(const char* str, const char* flag, double* value); -// Parses a string for a string flag, in the form of -// "--flag=value". +// Parses a string for a string flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseStringFlag(const char* str, const char* flag, std::string* value); +// Parses a string for a kvpairs flag in the form "--flag=key=value,key=value" +// +// On success, stores the value of the flag in *value and returns true. On +// failure returns false, though *value may have been mutated. +bool ParseKeyValueFlag(const char* str, const char* flag, + std::map* value); + // Returns true if the string matches the flag. bool IsFlag(const char* str, const char* flag); diff --git a/src/reporter.cc b/src/reporter.cc index 337575a118..b7d64fb1d3 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -22,8 +22,11 @@ #include #include "check.h" +#include "commandlineflags.h" #include "string_util.h" +DECLARE_kvpairs(benchmark_context); + namespace benchmark { BenchmarkReporter::BenchmarkReporter() @@ -64,6 +67,10 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } + for (const auto& kv: FLAGS_benchmark_context) { + Out << kv.first << ": " << kv.second << "\n"; + } + if (CPUInfo::Scaling::ENABLED == info.scaling) { Out << "***WARNING*** CPU scaling is enabled, the benchmark " "real time measurements may be noisy and will incur extra " diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc index 656020f2ec..8412008ffe 100644 --- a/test/commandlineflags_gtest.cc +++ b/test/commandlineflags_gtest.cc @@ -2,6 +2,7 @@ #include "../src/commandlineflags.h" #include "../src/internal_macros.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { @@ -19,9 +20,7 @@ int setenv(const char* name, const char* value, int overwrite) { return _putenv_s(name, value); } -int unsetenv(const char* name) { - return _putenv_s(name, ""); -} +int unsetenv(const char* name) { return _putenv_s(name, ""); } #endif // BENCHMARK_OS_WINDOWS @@ -197,5 +196,33 @@ TEST(StringFromEnv, Valid) { unsetenv("IN_ENV"); } +TEST(KvPairsFromEnv, Default) { + ASSERT_EQ(unsetenv("NOT_IN_ENV"), 0); + EXPECT_THAT(KvPairsFromEnv("not_in_env", {{"foo", "bar"}}), + testing::ElementsAre(testing::Pair("foo", "bar"))); +} + +TEST(KvPairsFromEnv, MalformedReturnsDefault) { + ASSERT_EQ(setenv("IN_ENV", "foo", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {{"foo", "bar"}}), + testing::ElementsAre(testing::Pair("foo", "bar"))); + unsetenv("IN_ENV"); +} + +TEST(KvPairsFromEnv, Single) { + ASSERT_EQ(setenv("IN_ENV", "foo=bar", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {}), + testing::ElementsAre(testing::Pair("foo", "bar"))); + unsetenv("IN_ENV"); +} + +TEST(KvPairsFromEnv, Multiple) { + ASSERT_EQ(setenv("IN_ENV", "foo=bar,baz=qux", 1), 0); + EXPECT_THAT(KvPairsFromEnv("in_env", {}), + testing::UnorderedElementsAre(testing::Pair("foo", "bar"), + testing::Pair("baz", "qux"))); + unsetenv("IN_ENV"); +} + } // namespace } // namespace benchmark From d0c227ccfd87de776f0a6e097b0d6039315608a2 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Wed, 5 May 2021 12:08:23 +0100 Subject: [PATCH 287/330] Add API to benchmark allowing for custom context to be added (#1137) * Add API to benchmark allowing for custom context to be added Fixes #525 * add docs * Add context flag output to JSON reporter * Plumb everything into the global context. * Add googletests for custom context * update docs with duplicate key behaviour --- README.md | 9 +++++++++ include/benchmark/benchmark.h | 3 +++ src/benchmark.cc | 28 ++++++++++++++++++++++------ src/json_reporter.cc | 10 ++++++++++ src/reporter.cc | 14 +++++++++----- test/benchmark_gtest.cc | 29 +++++++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 478ac89675..a853115b36 100644 --- a/README.md +++ b/README.md @@ -463,6 +463,15 @@ BM_memcpy/32 11 ns 11 ns 79545455 BM_memcpy/32k 2181 ns 2185 ns 324074 ``` +You can get the same effect with the API: + +```c++ + benchmark::AddCustomContext("foo", "bar"); +``` + +Note that attempts to add a second value with the same key will fail with an +error message. + ### Runtime and Reporting Considerations diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 664422b347..efcc0030b4 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -294,6 +294,9 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, // allocation measurements for benchmark runs. void RegisterMemoryManager(MemoryManager* memory_manager); +// Add a key-value pair to output as part of the context stanza in the report. +void AddCustomContext(const std::string& key, const std::string& value); + namespace internal { class Benchmark; class BenchmarkImp; diff --git a/src/benchmark.cc b/src/benchmark.cc index 930a75671a..16c347d605 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -105,10 +106,6 @@ DEFINE_string(benchmark_color, "auto"); // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. DEFINE_bool(benchmark_counters_tabular, false); -// Extra context to include in the output formatted as comma-separated key-value -// pairs. -DEFINE_kvpairs(benchmark_context, {}); - // The level of verbose logging to output DEFINE_int32(v, 0); @@ -117,9 +114,14 @@ DEFINE_int32(v, 0); DEFINE_string(benchmark_perf_counters, ""); namespace benchmark { - namespace internal { +// Extra context to include in the output formatted as comma-separated key-value +// pairs. Kept internal as it's only used for parsing from env/command line. +DEFINE_kvpairs(benchmark_context, {}); + +std::map* global_context = nullptr; + // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} @@ -435,6 +437,16 @@ void RegisterMemoryManager(MemoryManager* manager) { internal::memory_manager = manager; } +void AddCustomContext(const std::string& key, const std::string& value) { + if (internal::global_context == nullptr) { + internal::global_context = new std::map(); + } + if (!internal::global_context->emplace(key, value).second) { + std::cerr << "Failed to add custom context \"" << key << "\" as it already " + << "exists with value \"" << value << "\"\n"; + } +} + namespace internal { void PrintUsageAndExit() { @@ -496,13 +508,17 @@ void ParseCommandLineFlags(int* argc, char** argv) { } } for (auto const* flag : - {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) + {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) { if (*flag != "console" && *flag != "json" && *flag != "csv") { PrintUsageAndExit(); } + } if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } + for (const auto& kv : FLAGS_benchmark_context) { + AddCustomContext(kv.first, kv.second); + } } int InitializeStreams() { diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 959d245a34..effa6bd12e 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -29,6 +29,9 @@ #include "timers.h" namespace benchmark { +namespace internal { +extern std::map* global_context; +} namespace { @@ -160,6 +163,13 @@ bool JSONReporter::ReportContext(const Context& context) { const char build_type[] = "debug"; #endif out << indent << FormatKV("library_build_type", build_type) << "\n"; + + if (internal::global_context != nullptr) { + for (const auto& kv: *internal::global_context) { + out << indent << FormatKV(kv.first, kv.second) << "\n"; + } + } + // Close context block and open the list of benchmarks. out << inner_indent << "},\n"; out << inner_indent << "\"benchmarks\": [\n"; diff --git a/src/reporter.cc b/src/reporter.cc index b7d64fb1d3..14dd40dc72 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -18,16 +18,18 @@ #include #include +#include +#include #include #include #include "check.h" -#include "commandlineflags.h" #include "string_util.h" -DECLARE_kvpairs(benchmark_context); - namespace benchmark { +namespace internal { +extern std::map* global_context; +} BenchmarkReporter::BenchmarkReporter() : output_stream_(&std::cout), error_stream_(&std::cerr) {} @@ -67,8 +69,10 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } - for (const auto& kv: FLAGS_benchmark_context) { - Out << kv.first << ": " << kv.second << "\n"; + if (internal::global_context != nullptr) { + for (const auto& kv: *internal::global_context) { + Out << kv.first << ": " << kv.second << "\n"; + } } if (CPUInfo::Scaling::ENABLED == info.scaling) { diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index 6dbf7a5042..cbbf48b3b4 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -1,3 +1,5 @@ +#include +#include #include #include "../src/benchmark_register.h" @@ -6,6 +8,8 @@ namespace benchmark { namespace internal { +extern std::map* global_context; + namespace { TEST(AddRangeTest, Simple) { @@ -129,6 +133,31 @@ TEST(AddRangeTest, Simple8) { EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); } +TEST(AddCustomContext, Simple) { + EXPECT_THAT(global_context, nullptr); + + AddCustomContext("foo", "bar"); + AddCustomContext("baz", "qux"); + + EXPECT_THAT(*global_context, + testing::UnorderedElementsAre(testing::Pair("foo", "bar"), + testing::Pair("baz", "qux"))); + + global_context = nullptr; +} + +TEST(AddCustomContext, DuplicateKey) { + EXPECT_THAT(global_context, nullptr); + + AddCustomContext("foo", "bar"); + AddCustomContext("foo", "qux"); + + EXPECT_THAT(*global_context, + testing::UnorderedElementsAre(testing::Pair("foo", "bar"))); + + global_context = nullptr; +} + } // namespace } // namespace internal } // namespace benchmark From e50b572e899e0163ccc6f6bb8282ae4a123a8a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 6 May 2021 14:48:48 +0200 Subject: [PATCH 288/330] cmake: Add explicit BENCHMARK_ENABLE_LIBPFM option (#1141) --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e0f251fe5..77b3723971 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree buildi # in cases where it is not possible to build or find a valid version of gtest. option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON) +option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF) function(should_enable_assembly_tests) From e0826edea795ee1aac5e122c9632fa6e3556561b Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Thu, 6 May 2021 11:12:36 -0700 Subject: [PATCH 289/330] Fix StrSplit empty string case (#1142) This also fixes #1135. Because StrSplit was returning a vector with an empty string, it was treated by PerfCounters::Create as a legitimate ask for setting up a counter with that name. The empty vector is understood by PerfCounters as "just return NoCounters()". --- src/string_util.cc | 1 + test/string_util_gtest.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/string_util.cc b/src/string_util.cc index 53b1532b94..cb973a7c6f 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -164,6 +164,7 @@ std::string StrFormat(const char* format, ...) { } std::vector StrSplit(const std::string& str, char delim) { + if (str.empty()) return {}; std::vector ret; size_t first = 0; size_t next = str.find(delim); diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 77a719a69d..1ad9fb9d26 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -151,7 +151,7 @@ TEST(StringUtilTest, stod) { } TEST(StringUtilTest, StrSplit) { - EXPECT_EQ(benchmark::StrSplit("", ','), std::vector{""}); + EXPECT_EQ(benchmark::StrSplit("", ','), std::vector{}); EXPECT_EQ(benchmark::StrSplit("hello", ','), std::vector({"hello"})); EXPECT_EQ(benchmark::StrSplit("hello,there", ','), From a2e8a8a9db0d842929514377759ae83da8a76da6 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 6 May 2021 22:31:14 +0300 Subject: [PATCH 290/330] Clean -Wreserved-identifier instances (#1143) --- include/benchmark/benchmark.h | 2 +- src/benchmark_register.cc | 6 +---- src/mutex.h | 42 +++++++++++++++++------------------ test/output_test.h | 12 +++++----- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index efcc0030b4..858fcfb6ca 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1123,7 +1123,7 @@ class Fixture : public internal::Benchmark { // Helpers for generating unique variable names #define BENCHMARK_PRIVATE_NAME(n) \ - BENCHMARK_PRIVATE_CONCAT(_benchmark_, BENCHMARK_PRIVATE_UNIQUE_ID, n) + BENCHMARK_PRIVATE_CONCAT(benchmark_uniq_, BENCHMARK_PRIVATE_UNIQUE_ID, n) #define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) #define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c // Helper for concatenation with macro name expansion diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index c007876f24..d62441cfe8 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -35,11 +36,6 @@ #include #include -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif -#include - #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" #include "check.h" diff --git a/src/mutex.h b/src/mutex.h index 3fac79aea4..9cc414ec46 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -9,60 +9,60 @@ // Enable thread safety attributes only with clang. // The attributes can be safely erased when compiling with other compilers. #if defined(HAVE_THREAD_SAFETY_ATTRIBUTES) -#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#define THREAD_ANNOTATION_ATTRIBUTE_(x) __attribute__((x)) #else -#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#define THREAD_ANNOTATION_ATTRIBUTE_(x) // no-op #endif -#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x)) +#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(capability(x)) -#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) +#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE_(scoped_lockable) -#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE_(guarded_by(x)) -#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) +#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE_(pt_guarded_by(x)) #define ACQUIRED_BEFORE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(acquired_before(__VA_ARGS__)) #define ACQUIRED_AFTER(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(acquired_after(__VA_ARGS__)) #define REQUIRES(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(requires_capability(__VA_ARGS__)) #define REQUIRES_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(requires_shared_capability(__VA_ARGS__)) #define ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(acquire_capability(__VA_ARGS__)) #define ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(acquire_shared_capability(__VA_ARGS__)) #define RELEASE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(release_capability(__VA_ARGS__)) #define RELEASE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(release_shared_capability(__VA_ARGS__)) #define TRY_ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(try_acquire_capability(__VA_ARGS__)) #define TRY_ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__)) + THREAD_ANNOTATION_ATTRIBUTE_(try_acquire_shared_capability(__VA_ARGS__)) -#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) +#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE_(locks_excluded(__VA_ARGS__)) -#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x)) +#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(assert_capability(x)) #define ASSERT_SHARED_CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x)) + THREAD_ANNOTATION_ATTRIBUTE_(assert_shared_capability(x)) -#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) +#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE_(lock_returned(x)) #define NO_THREAD_SAFETY_ANALYSIS \ - THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + THREAD_ANNOTATION_ATTRIBUTE_(no_thread_safety_analysis) namespace benchmark { diff --git a/test/output_test.h b/test/output_test.h index 9385761b21..15368f9b68 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -158,7 +158,7 @@ T Results::GetAs(const char* entry_name) const { // clang-format off -#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ +#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \ CONCAT(CHECK_, relationship) \ (entry.getfn< var_type >(var_name), (value)) << "\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ @@ -169,7 +169,7 @@ T Results::GetAs(const char* entry_name) const { // check with tolerance. eps_factor is the tolerance window, which is // interpreted relative to value (eg, 0.1 means 10% of value). -#define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ +#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ CONCAT(CHECK_FLOAT_, relationship) \ (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ @@ -187,16 +187,16 @@ T Results::GetAs(const char* entry_name) const { << "%)" #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ - _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) + CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value) #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ - _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) + CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value) #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \ - _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor) + CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor) #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ - _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) + CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) // clang-format on From a53b8853aa329825d1177e35416e1b4107d81879 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 7 May 2021 12:39:04 +0100 Subject: [PATCH 291/330] Add ubuntu-14.04 build and test workflow (#1131) * Add ubuntu-14.04 build and test workflow * avoid '.' in job name * no need for fail fast * fix workflow syntax * install some stuff * better compiler installations * update before install * just say yes * trying to match up some paths * Difference between runner and github context in container? * Try some judicious logging * cmake 3.5+ required * specific compiler versions * need git for googletest * Disable testing on old compilers * disable testing properly --- .github/workflows/build-and-test.yml | 45 ++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 731a19a364..f1f8cdb452 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -7,7 +7,6 @@ on: branches: [ master ] jobs: - # TODO: add job using container ubuntu:14.04 with and without g++-6 # TODO: add 32-bit builds (g++ and clang++) for ubuntu (requires g++-multilib and libc6:i386) # TODO: add coverage build (requires lcov) # TODO: add clang + libc++ builds for ubuntu @@ -40,7 +39,7 @@ jobs: shell: bash working-directory: ${{ runner.workspace }}/_build run: > - cmake $GITHUB_WORKSPACE + cmake $GITHUB_WORKSPACE -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} @@ -53,3 +52,45 @@ jobs: shell: bash working-directory: ${{ runner.workspace }}/_build run: ctest -C ${{ matrix.build_type }} -VV + + # TODO: add compiler g++-6 to ubuntu-14.04 job + # TODO: enable testing for g++-6 compiler + ubuntu-14_04: + name: ubuntu-14.04.${{ matrix.build_type }}.${{ matrix.compiler }} + runs-on: [ubuntu-latest] + strategy: + fail-fast: false + matrix: + build_type: ['Release', 'Debug'] + compiler: [g++-4.8, clang++-3.6] + container: ubuntu:14.04 + steps: + - uses: actions/checkout@v2 + + - name: install required bits + run: sudo apt update && sudo apt -y install clang-3.6 cmake3 g++-4.8 git + + - name: create build environment + run: cmake -E make_directory $GITHUB_WORKSPACE/_build + + - name: configure cmake + env: + CXX: ${{ matrix.compiler }} + shell: bash + working-directory: ${{ github.workspace }}/_build + run: > + cmake $GITHUB_WORKSPACE + -DBENCHMARK_ENABLE_TESTING=off + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + #-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + + - name: build + shell: bash + working-directory: ${{ github.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} + + # - name: test + # shell: bash + # working-directory: ${{ github.workspace }}/_build + # run: ctest -C ${{ matrix.build_type }} -VV + From 17948a78ee4137e15fcdbfecdfee27f8eca279f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Janiszewski?= Date: Fri, 7 May 2021 15:08:24 +0200 Subject: [PATCH 292/330] Add MSVC ARM64 support to cmake (#1090) --- CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 77b3723971..9e68d39188 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,17 @@ option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +if(MSVC) + # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and + # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the + # undocumented, but working variable. + # See https://gitlab.kitware.com/cmake/cmake/-/issues/15170 + set(CMAKE_SYSTEM_PROCESSOR ${MSVC_CXX_ARCHITECTURE_ID}) + if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "ARM") + set(CMAKE_CROSSCOMPILING TRUE) + endif() +endif() + set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF) function(should_enable_assembly_tests) if(CMAKE_BUILD_TYPE) From 551a21bad0ba349a0b0434d4276547b26f8d5d01 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 7 May 2021 14:23:05 +0100 Subject: [PATCH 293/330] add g++-6 to ubuntu-14.04 (#1144) * add g++-6 to ubuntu-14.04 * fix syntax * fix yamllint errors for build-and-test * fix 'add-apt-repository' command not found * make 'run tests' explicit * enable testing and run both release and debug * oops --- .github/workflows/build-and-test.yml | 110 ++++++++++++++++----------- 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f1f8cdb452..5a5a101acc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,12 +2,13 @@ name: build-and-test on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: - # TODO: add 32-bit builds (g++ and clang++) for ubuntu (requires g++-multilib and libc6:i386) + # TODO: add 32-bit builds (g++ and clang++) for ubuntu + # (requires g++-multilib and libc6:i386) # TODO: add coverage build (requires lcov) # TODO: add clang + libc++ builds for ubuntu # TODO: add clang + ubsan/asan/msan + libc++ builds for ubuntu @@ -28,30 +29,30 @@ jobs: os: windows-latest build_type: 'Debug' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: create build environment - run: cmake -E make_directory ${{ runner.workspace }}/_build + - name: create build environment + run: cmake -E make_directory ${{ runner.workspace }}/_build - - name: configure cmake - env: - CXX: ${{ matrix.compiler }} - shell: bash - working-directory: ${{ runner.workspace }}/_build - run: > - cmake $GITHUB_WORKSPACE - -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + - name: configure cmake + env: + CXX: ${{ matrix.compiler }} + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: > + cmake $GITHUB_WORKSPACE + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - - name: build - shell: bash - working-directory: ${{ runner.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} + - name: build + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} - - name: test - shell: bash - working-directory: ${{ runner.workspace }}/_build - run: ctest -C ${{ matrix.build_type }} -VV + - name: test + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: ctest -C ${{ matrix.build_type }} -VV # TODO: add compiler g++-6 to ubuntu-14.04 job # TODO: enable testing for g++-6 compiler @@ -63,34 +64,51 @@ jobs: matrix: build_type: ['Release', 'Debug'] compiler: [g++-4.8, clang++-3.6] + include: + - compiler: g++-6 + build_type: 'Debug' + run_tests: true + - compiler: g++-6 + build_type: 'Release' + run_tests: true container: ubuntu:14.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: install required bits - run: sudo apt update && sudo apt -y install clang-3.6 cmake3 g++-4.8 git + - name: install required bits + run: | + sudo apt update + sudo apt -y install clang-3.6 cmake3 g++-4.8 git - - name: create build environment - run: cmake -E make_directory $GITHUB_WORKSPACE/_build + - name: install other bits + if: ${{ matrix.compiler }} == g++-6 + run: | + sudo apt -y install software-properties-common + sudo add-apt-repository -y "ppa:ubuntu-toolchain-r/test" + sudo apt update + sudo apt -y install g++-6 - - name: configure cmake - env: - CXX: ${{ matrix.compiler }} - shell: bash - working-directory: ${{ github.workspace }}/_build - run: > - cmake $GITHUB_WORKSPACE - -DBENCHMARK_ENABLE_TESTING=off - -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - #-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + - name: create build environment + run: cmake -E make_directory $GITHUB_WORKSPACE/_build - - name: build - shell: bash - working-directory: ${{ github.workspace }}/_build - run: cmake --build . --config ${{ matrix.build_type }} + - name: configure cmake + env: + CXX: ${{ matrix.compiler }} + shell: bash + working-directory: ${{ github.workspace }}/_build + run: > + cmake $GITHUB_WORKSPACE + -DBENCHMARK_ENABLE_TESTING=${{ matrix.run_tests }} + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=${{ matrix.run_tests }} - # - name: test - # shell: bash - # working-directory: ${{ github.workspace }}/_build - # run: ctest -C ${{ matrix.build_type }} -VV + - name: build + shell: bash + working-directory: ${{ github.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} + - name: test + if: ${{ matrix.run_tests }} + shell: bash + working-directory: ${{ github.workspace }}/_build + run: ctest -C ${{ matrix.build_type }} -VV From 0852c79f3511dec3115e72d19d99ec8146404cc5 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 7 May 2021 14:24:00 +0100 Subject: [PATCH 294/330] remove done TODOs --- .github/workflows/build-and-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5a5a101acc..955da5f9b7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -54,8 +54,6 @@ jobs: working-directory: ${{ runner.workspace }}/_build run: ctest -C ${{ matrix.build_type }} -VV - # TODO: add compiler g++-6 to ubuntu-14.04 job - # TODO: enable testing for g++-6 compiler ubuntu-14_04: name: ubuntu-14.04.${{ matrix.build_type }}.${{ matrix.compiler }} runs-on: [ubuntu-latest] From 1f47b6b64cfe4d5e712247e5ca5ecba4d42faf8d Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 10 May 2021 13:43:08 +0100 Subject: [PATCH 295/330] Remove travis configs that are covered by actions (#1145) --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36e343dbfe..4407af8d31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,6 @@ matrix: packages: - lcov env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Coverage - - compiler: gcc - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug - - compiler: gcc - env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release - compiler: gcc addons: apt: @@ -44,10 +40,6 @@ matrix: - COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold" - - compiler: clang - env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Debug - - compiler: clang - env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release # Clang w/ libc++ - compiler: clang dist: xenial @@ -146,16 +138,6 @@ matrix: - ENABLE_SANITIZER=1 - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" - EXTRA_CXX_FLAGS="-stdlib=libc++" - - os: osx - osx_image: xcode8.3 - compiler: clang - env: - - COMPILER=clang++ BUILD_TYPE=Debug - - os: osx - osx_image: xcode8.3 - compiler: clang - env: - - COMPILER=clang++ BUILD_TYPE=Release - os: osx osx_image: xcode8.3 compiler: clang @@ -164,11 +146,6 @@ matrix: - BUILD_TYPE=Release - BUILD_32_BITS=ON - EXTRA_FLAGS="-m32" - - os: osx - osx_image: xcode9.4 - compiler: gcc - env: - - COMPILER=g++-7 C_COMPILER=gcc-7 BUILD_TYPE=Debug before_script: - if [ -n "${LIBCXX_BUILD}" ]; then From 3b508fad1f9fbc970dad0a190119ae6284cad855 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Mon, 10 May 2021 17:12:09 +0100 Subject: [PATCH 296/330] Refactor `BenchmarkInstance` (#1148) * Refactor BenchmarkInstance (precursor to #1105) * fix bazel (debug) build * clang-format on header * fix build error on g++-4.8 --- include/benchmark/benchmark.h | 5 ++- src/benchmark.cc | 8 ++-- src/benchmark_api_internal.cc | 82 +++++++++++++++++++++++++++++++++-- src/benchmark_api_internal.h | 63 ++++++++++++++++++--------- src/benchmark_register.cc | 71 +----------------------------- src/benchmark_runner.cc | 60 ++++++++++++------------- 6 files changed, 159 insertions(+), 130 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 858fcfb6ca..4c70c3a0e2 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -448,7 +448,7 @@ struct Statistics { : name_(name), compute_(compute) {} }; -struct BenchmarkInstance; +class BenchmarkInstance; class ThreadTimer; class ThreadManager; class PerfCountersMeasurement; @@ -703,7 +703,7 @@ class State { internal::ThreadManager* const manager_; internal::PerfCountersMeasurement* const perf_counters_measurement_; - friend struct internal::BenchmarkInstance; + friend class internal::BenchmarkInstance; }; inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunning() { @@ -981,6 +981,7 @@ class Benchmark { private: friend class BenchmarkFamilies; + friend class BenchmarkInstance; std::string name_; AggregationReportMode aggregation_report_mode_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 16c347d605..d205232b8a 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -253,10 +253,10 @@ void RunBenchmarks(const std::vector& benchmarks, size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { name_field_width = - std::max(name_field_width, benchmark.name.str().size()); - might_have_aggregates |= benchmark.repetitions > 1; + std::max(name_field_width, benchmark.name().str().size()); + might_have_aggregates |= benchmark.repetitions() > 1; - for (const auto& Stat : *benchmark.statistics) + for (const auto& Stat : benchmark.statistics()) stat_field_width = std::max(stat_field_width, Stat.name_.size()); } if (might_have_aggregates) name_field_width += 1 + stat_field_width; @@ -425,7 +425,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, if (FLAGS_benchmark_list_tests) { for (auto const& benchmark : benchmarks) - Out << benchmark.name.str() << "\n"; + Out << benchmark.name().str() << "\n"; } else { internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 804ef894ab..553ff44e5c 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -1,17 +1,91 @@ #include "benchmark_api_internal.h" +#include + +#include "string_util.h" + namespace benchmark { namespace internal { +BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, + const std::vector& args, + int thread_count) + : benchmark_(*benchmark), + aggregation_report_mode_(benchmark_.aggregation_report_mode_), + args_(args), + time_unit_(benchmark_.time_unit_), + measure_process_cpu_time_(benchmark_.measure_process_cpu_time_), + use_real_time_(benchmark_.use_real_time_), + use_manual_time_(benchmark_.use_manual_time_), + complexity_(benchmark_.complexity_), + complexity_lambda_(benchmark_.complexity_lambda_), + statistics_(benchmark_.statistics_), + repetitions_(benchmark_.repetitions_), + min_time_(benchmark_.min_time_), + iterations_(benchmark_.iterations_), + threads_(thread_count) { + name_.function_name = benchmark_.name_; + + size_t arg_i = 0; + for (const auto& arg : args) { + if (!name_.args.empty()) { + name_.args += '/'; + } + + if (arg_i < benchmark->arg_names_.size()) { + const auto& arg_name = benchmark_.arg_names_[arg_i]; + if (!arg_name.empty()) { + name_.args += StrFormat("%s:", arg_name.c_str()); + } + } + + name_.args += StrFormat("%" PRId64, arg); + ++arg_i; + } + + if (!IsZero(benchmark->min_time_)) { + name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); + } + + if (benchmark_.iterations_ != 0) { + name_.iterations = StrFormat( + "iterations:%lu", static_cast(benchmark_.iterations_)); + } + + if (benchmark_.repetitions_ != 0) { + name_.repetitions = StrFormat("repeats:%d", benchmark_.repetitions_); + } + + if (benchmark_.measure_process_cpu_time_) { + name_.time_type = "process_time"; + } + + if (benchmark_.use_manual_time_) { + if (!name_.time_type.empty()) { + name_.time_type += '/'; + } + name_.time_type += "manual_time"; + } else if (benchmark_.use_real_time_) { + if (!name_.time_type.empty()) { + name_.time_type += '/'; + } + name_.time_type += "real_time"; + } + + if (!benchmark_.thread_counts_.empty()) { + name_.threads = StrFormat("threads:%d", threads_); + } +} + State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const { - State st(iters, arg, thread_id, threads, timer, manager, + State st(iters, args_, thread_id, threads_, timer, manager, perf_counters_measurement); - benchmark->Run(st); + benchmark_.Run(st); return st; } -} // internal -} // benchmark +} // namespace internal +} // namespace benchmark diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index b740bce115..b0595e5a12 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -1,9 +1,6 @@ #ifndef BENCHMARK_API_INTERNAL_H #define BENCHMARK_API_INTERNAL_H -#include "benchmark/benchmark.h" -#include "commandlineflags.h" - #include #include #include @@ -11,33 +8,57 @@ #include #include +#include "benchmark/benchmark.h" +#include "commandlineflags.h" + namespace benchmark { namespace internal { // Information kept per benchmark we may want to run -struct BenchmarkInstance { - BenchmarkName name; - Benchmark* benchmark; - AggregationReportMode aggregation_report_mode; - std::vector arg; - TimeUnit time_unit; - int range_multiplier; - bool measure_process_cpu_time; - bool use_real_time; - bool use_manual_time; - BigO complexity; - BigOFunc* complexity_lambda; - UserCounters counters; - const std::vector* statistics; +class BenchmarkInstance { + public: + BenchmarkInstance(Benchmark* benchmark, const std::vector& args, + int threads); + + const BenchmarkName& name() const { return name_; } + AggregationReportMode aggregation_report_mode() const { + return aggregation_report_mode_; + } + TimeUnit time_unit() const { return time_unit_; } + bool measure_process_cpu_time() const { return measure_process_cpu_time_; } + bool use_real_time() const { return use_real_time_; } + bool use_manual_time() const { return use_manual_time_; } + BigO complexity() const { return complexity_; } + BigOFunc& complexity_lambda() const { return *complexity_lambda_; } + const std::vector& statistics() const { return statistics_; } + int repetitions() const { return repetitions_; } + double min_time() const { return min_time_; } + IterationCount iterations() const { return iterations_; } + int threads() const { return threads_; } + bool last_benchmark_instance; - int repetitions; - double min_time; - IterationCount iterations; - int threads; // Number of concurrent threads to us State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const; + + private: + BenchmarkName name_; + Benchmark& benchmark_; + AggregationReportMode aggregation_report_mode_; + const std::vector& args_; + TimeUnit time_unit_; + bool measure_process_cpu_time_; + bool use_real_time_; + bool use_manual_time_; + BigO complexity_; + BigOFunc* complexity_lambda_; + UserCounters counters_; + const std::vector& statistics_; + int repetitions_; + double min_time_; + IterationCount iterations_; + int threads_; // Number of concurrent threads to us }; bool FindBenchmarksInternal(const std::string& re, diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index d62441cfe8..1f0dcd1d0e 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -154,76 +154,9 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { - BenchmarkInstance instance; - instance.name.function_name = family->name_; - instance.benchmark = family.get(); - instance.aggregation_report_mode = family->aggregation_report_mode_; - instance.arg = args; - instance.time_unit = family->time_unit_; - instance.range_multiplier = family->range_multiplier_; - instance.min_time = family->min_time_; - instance.iterations = family->iterations_; - instance.repetitions = family->repetitions_; - instance.measure_process_cpu_time = family->measure_process_cpu_time_; - instance.use_real_time = family->use_real_time_; - instance.use_manual_time = family->use_manual_time_; - instance.complexity = family->complexity_; - instance.complexity_lambda = family->complexity_lambda_; - instance.statistics = &family->statistics_; - instance.threads = num_threads; - - // Add arguments to instance name - size_t arg_i = 0; - for (auto const& arg : args) { - if (!instance.name.args.empty()) { - instance.name.args += '/'; - } - - if (arg_i < family->arg_names_.size()) { - const auto& arg_name = family->arg_names_[arg_i]; - if (!arg_name.empty()) { - instance.name.args += StrFormat("%s:", arg_name.c_str()); - } - } - - instance.name.args += StrFormat("%" PRId64, arg); - ++arg_i; - } - - if (!IsZero(family->min_time_)) - instance.name.min_time = - StrFormat("min_time:%0.3f", family->min_time_); - if (family->iterations_ != 0) { - instance.name.iterations = - StrFormat("iterations:%lu", - static_cast(family->iterations_)); - } - if (family->repetitions_ != 0) - instance.name.repetitions = - StrFormat("repeats:%d", family->repetitions_); - - if (family->measure_process_cpu_time_) { - instance.name.time_type = "process_time"; - } - - if (family->use_manual_time_) { - if (!instance.name.time_type.empty()) { - instance.name.time_type += '/'; - } - instance.name.time_type += "manual_time"; - } else if (family->use_real_time_) { - if (!instance.name.time_type.empty()) { - instance.name.time_type += '/'; - } - instance.name.time_type += "real_time"; - } - - // Add the number of threads used to the name - if (!family->thread_counts_.empty()) { - instance.name.threads = StrFormat("threads:%d", instance.threads); - } + BenchmarkInstance instance(family.get(), args, num_threads); - const auto full_name = instance.name.str(); + const auto full_name = instance.name().str(); if ((re.Match(full_name) && !isNegativeFilter) || (!re.Match(full_name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 083d18492a..7f3cd367a1 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -71,28 +71,28 @@ BenchmarkReporter::Run CreateRunReport( // Create report about this benchmark run. BenchmarkReporter::Run report; - report.run_name = b.name; + report.run_name = b.name(); report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; // This is the total iterations across all threads. report.iterations = results.iterations; - report.time_unit = b.time_unit; - report.threads = b.threads; + report.time_unit = b.time_unit(); + report.threads = b.threads(); report.repetition_index = repetition_index; - report.repetitions = b.repetitions; + report.repetitions = b.repetitions(); if (!report.error_occurred) { - if (b.use_manual_time) { + if (b.use_manual_time()) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } report.cpu_accumulated_time = results.cpu_time_used; report.complexity_n = results.complexity_n; - report.complexity = b.complexity; - report.complexity_lambda = b.complexity_lambda; - report.statistics = b.statistics; + report.complexity = b.complexity(); + report.complexity_lambda = b.complexity_lambda(); + report.statistics = &b.statistics(); report.counters = results.counters; if (memory_iterations > 0) { @@ -104,7 +104,7 @@ BenchmarkReporter::Run CreateRunReport( report.max_bytes_used = memory_result.max_bytes_used; } - internal::Finish(&report.counters, results.iterations, seconds, b.threads); + internal::Finish(&report.counters, results.iterations, seconds, b.threads()); } return report; } @@ -115,7 +115,7 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, int thread_id, ThreadManager* manager, PerfCountersMeasurement* perf_counters_measurement) { internal::ThreadTimer timer( - b->measure_process_cpu_time + b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); State st = @@ -141,12 +141,12 @@ class BenchmarkRunner { std::vector* complexity_reports_) : b(b_), complexity_reports(*complexity_reports_), - min_time(!IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time), - repeats(b.repetitions != 0 ? b.repetitions + min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations != 0), - pool(b.threads - 1), - iters(has_explicit_iteration_count ? b.iterations : 1), + has_explicit_iteration_count(b.iterations() != 0), + pool(b.threads() - 1), + iters(has_explicit_iteration_count ? b.iterations() : 1), perf_counters_measurement( PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), perf_counters_measurement_ptr(perf_counters_measurement.IsValid() @@ -157,13 +157,13 @@ class BenchmarkRunner { FLAGS_benchmark_display_aggregates_only); run_results.file_report_aggregates_only = FLAGS_benchmark_report_aggregates_only; - if (b.aggregation_report_mode != internal::ARM_Unspecified) { + if (b.aggregation_report_mode() != internal::ARM_Unspecified) { run_results.display_report_aggregates_only = - (b.aggregation_report_mode & + (b.aggregation_report_mode() & internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = - (b.aggregation_report_mode & internal::ARM_FileReportAggregatesOnly); - CHECK(b.threads == 1 || !perf_counters_measurement.IsValid()) + (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); + CHECK(b.threads() == 1 || !perf_counters_measurement.IsValid()) << "Perf counters are not supported in multi-threaded cases.\n"; CHECK(FLAGS_benchmark_perf_counters.empty() || perf_counters_measurement.IsValid()) @@ -178,7 +178,7 @@ class BenchmarkRunner { run_results.aggregates_only = ComputeStats(run_results.non_aggregates); // Maybe calculate complexity report - if ((b.complexity != oNone) && b.last_benchmark_instance) { + if ((b.complexity() != oNone) && b.last_benchmark_instance) { auto additional_run_stats = ComputeBigO(complexity_reports); run_results.aggregates_only.insert(run_results.aggregates_only.end(), additional_run_stats.begin(), @@ -214,10 +214,10 @@ class BenchmarkRunner { double seconds; }; IterationResults DoNIterations() { - VLOG(2) << "Running " << b.name.str() << " for " << iters << "\n"; + VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; std::unique_ptr manager; - manager.reset(new internal::ThreadManager(b.threads)); + manager.reset(new internal::ThreadManager(b.threads())); // Run all but one thread in separate threads for (std::size_t ti = 0; ti < pool.size(); ++ti) { @@ -244,23 +244,23 @@ class BenchmarkRunner { manager.reset(); // Adjust real/manual time stats since they were reported per thread. - i.results.real_time_used /= b.threads; - i.results.manual_time_used /= b.threads; + i.results.real_time_used /= b.threads(); + i.results.manual_time_used /= b.threads(); // If we were measuring whole-process CPU usage, adjust the CPU time too. - if (b.measure_process_cpu_time) i.results.cpu_time_used /= b.threads; + if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" << i.results.real_time_used << "\n"; // By using KeepRunningBatch a benchmark can iterate more times than // requested, so take the iteration count from i.results. - i.iters = i.results.iterations / b.threads; + i.iters = i.results.iterations / b.threads(); // Base decisions off of real time if requested by this benchmark. i.seconds = i.results.cpu_time_used; - if (b.use_manual_time) { + if (b.use_manual_time()) { i.seconds = i.results.manual_time_used; - } else if (b.use_real_time) { + } else if (b.use_real_time()) { i.seconds = i.results.real_time_used; } @@ -301,7 +301,7 @@ class BenchmarkRunner { // CPU time is specified but the elapsed real time greatly exceeds // the minimum time. // Note that user provided timers are except from this sanity check. - ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time); + ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); } void DoOneRepetition(int64_t repetition_index) { @@ -360,7 +360,7 @@ class BenchmarkRunner { CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, repetition_index); - if (!report.error_occurred && b.complexity != oNone) + if (!report.error_occurred && b.complexity() != oNone) complexity_reports.push_back(report); run_results.non_aggregates.push_back(report); From 7d0d9061d83b663ce05d9de5da3d5865a3845b79 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Tue, 11 May 2021 13:56:00 +0200 Subject: [PATCH 297/330] Support -Wsuggest-override (#1059) * Support -Wsuggest-override google/benchmark is C++11 compatible but doesn't use the `override` keyword. Projects using google/benchmark with enabled `-Wsuggest-override` and `-Werror` will fail to compile. * Add -Wsuggest-override cxx flag * Revert unrelated formatting * Revert unrelated formatting, take 2 * Revert unrelated formatting, take 3 * Disable -Wsuggest-override when compiling tests, gtest does not handle it yet Co-authored-by: Dominic Hamon --- CMakeLists.txt | 6 ++- include/benchmark/benchmark.h | 82 ++++++++++++++++++--------------- test/args_product_test.cc | 2 +- test/filter_test.cc | 4 +- test/fixture_test.cc | 4 +- test/map_test.cc | 4 +- test/memory_manager_test.cc | 4 +- test/multiple_ranges_test.cc | 2 +- test/output_test_helper.cc | 6 +-- test/register_benchmark_test.cc | 2 +- test/skip_with_error_test.cc | 4 +- 11 files changed, 65 insertions(+), 55 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e68d39188..e75d41f4f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,10 @@ else() add_cxx_compiler_flag(-Werror RELEASE) add_cxx_compiler_flag(-Werror RELWITHDEBINFO) add_cxx_compiler_flag(-Werror MINSIZEREL) + if (NOT BENCHMARK_ENABLE_TESTING) + # Disable warning when compiling tests as gtest does not use 'override'. + add_cxx_compiler_flag(-Wsuggest-override) + endif() # Disabled until googletest (gmock) stops emitting variadic macro warnings #add_cxx_compiler_flag(-pedantic) #add_cxx_compiler_flag(-pedantic-errors) @@ -284,7 +288,7 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) if (BENCHMARK_ENABLE_LIBPFM) - find_package(PFM) + find_package(PFM) endif() # Set up directories diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 4c70c3a0e2..58d9237535 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -263,6 +263,12 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_UNREACHABLE() ((void)0) #endif +#ifdef BENCHMARK_HAS_CXX11 +#define BENCHMARK_OVERRIDE override +#else +#define BENCHMARK_OVERRIDE +#endif + namespace benchmark { class BenchmarkReporter; class MemoryManager; @@ -1029,7 +1035,7 @@ class FunctionBenchmark : public Benchmark { FunctionBenchmark(const char* name, Function* func) : Benchmark(name), func_(func) {} - virtual void Run(State& st); + virtual void Run(State& st) BENCHMARK_OVERRIDE; private: Function* func_; @@ -1039,7 +1045,7 @@ class FunctionBenchmark : public Benchmark { template class LambdaBenchmark : public Benchmark { public: - virtual void Run(State& st) { lambda_(st); } + virtual void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } private: template @@ -1091,7 +1097,7 @@ class Fixture : public internal::Benchmark { public: Fixture() : internal::Benchmark("") {} - virtual void Run(State& st) { + virtual void Run(State& st) BENCHMARK_OVERRIDE { this->SetUp(st); this->BenchmarkCase(st); this->TearDown(st); @@ -1199,37 +1205,37 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) #endif -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() : BaseClass() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&); \ +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() : BaseClass() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() : BaseClass() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&); \ +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() : BaseClass() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() : BaseClass() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&); \ +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() : BaseClass() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #ifdef BENCHMARK_HAS_CXX11 @@ -1241,7 +1247,7 @@ class Fixture : public internal::Benchmark { } \ \ protected: \ - virtual void BenchmarkCase(::benchmark::State&); \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #else #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ @@ -1533,8 +1539,8 @@ class ConsoleReporter : public BenchmarkReporter { prev_counters_(), printed_header_(false) {} - virtual bool ReportContext(const Context& context); - virtual void ReportRuns(const std::vector& reports); + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; protected: virtual void PrintRunData(const Run& report); @@ -1549,9 +1555,9 @@ class ConsoleReporter : public BenchmarkReporter { class JSONReporter : public BenchmarkReporter { public: JSONReporter() : first_report_(true) {} - virtual bool ReportContext(const Context& context); - virtual void ReportRuns(const std::vector& reports); - virtual void Finalize(); + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + virtual void Finalize() BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1564,8 +1570,8 @@ class BENCHMARK_DEPRECATED_MSG( : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} - virtual bool ReportContext(const Context& context); - virtual void ReportRuns(const std::vector& reports); + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); diff --git a/test/args_product_test.cc b/test/args_product_test.cc index 8a859f8415..32a75d50dd 100644 --- a/test/args_product_test.cc +++ b/test/args_product_test.cc @@ -23,7 +23,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { {2, 15, 10, 9}, {4, 5, 6, 11}}) {} - void SetUp(const ::benchmark::State& state) { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { std::vector ranges = {state.range(0), state.range(1), state.range(2), state.range(3)}; diff --git a/test/filter_test.cc b/test/filter_test.cc index 0e27065c15..fece6dc243 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -14,11 +14,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { ++count_; ConsoleReporter::ReportRuns(report); }; diff --git a/test/fixture_test.cc b/test/fixture_test.cc index a331c7de8d..eba0a42d9c 100644 --- a/test/fixture_test.cc +++ b/test/fixture_test.cc @@ -8,14 +8,14 @@ class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { if (state.thread_index == 0) { assert(data.get() == nullptr); data.reset(new int(42)); } } - void TearDown(const ::benchmark::State& state) { + void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { if (state.thread_index == 0) { assert(data.get() != nullptr); data.reset(); diff --git a/test/map_test.cc b/test/map_test.cc index dbf7982a36..86391b3601 100644 --- a/test/map_test.cc +++ b/test/map_test.cc @@ -34,11 +34,11 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); // Using fixtures. class MapFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& st) { + void SetUp(const ::benchmark::State& st) BENCHMARK_OVERRIDE { m = ConstructRandomMap(static_cast(st.range(0))); } - void TearDown(const ::benchmark::State&) { m.clear(); } + void TearDown(const ::benchmark::State&) BENCHMARK_OVERRIDE { m.clear(); } std::map m; }; diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 90bed16cff..71d4d0ec9c 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -5,8 +5,8 @@ #include "output_test.h" class TestMemoryManager : public benchmark::MemoryManager { - void Start() {} - void Stop(Result* result) { + void Start() BENCHMARK_OVERRIDE {} + void Stop(Result* result) BENCHMARK_OVERRIDE { result->num_allocs = 42; result->max_bytes_used = 42000; } diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc index b25f40eb52..6b61f3af47 100644 --- a/test/multiple_ranges_test.cc +++ b/test/multiple_ranges_test.cc @@ -28,7 +28,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {2, 7, 15}, {7, 6, 3}}) {} - void SetUp(const ::benchmark::State& state) { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { std::vector ranges = {state.range(0), state.range(1), state.range(2)}; diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index 1aebc55d01..b8ef120574 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -139,7 +139,7 @@ class TestReporter : public benchmark::BenchmarkReporter { TestReporter(std::vector reps) : reporters_(reps) {} - virtual bool ReportContext(const Context& context) { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { bool last_ret = false; bool first = true; for (auto rep : reporters_) { @@ -153,10 +153,10 @@ class TestReporter : public benchmark::BenchmarkReporter { return last_ret; } - void ReportRuns(const std::vector& report) { + void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { for (auto rep : reporters_) rep->ReportRuns(report); } - void Finalize() { + void Finalize() BENCHMARK_OVERRIDE { for (auto rep : reporters_) rep->Finalize(); } diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc index 3ac5b21fb3..c027eabaca 100644 --- a/test/register_benchmark_test.cc +++ b/test/register_benchmark_test.cc @@ -10,7 +10,7 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual void ReportRuns(const std::vector& report) { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 97a2e3c03b..827966e9df 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -10,11 +10,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } From e539e807daa0bc779bb6768b33ee4747624b4ec9 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Wed, 19 May 2021 01:49:05 -0700 Subject: [PATCH 298/330] [PFM] Extend perf counter support to multi-threaded cases. (#1153) * Extend perf counter support to multi-threaded cases. * Docs update * const-ed Snapshot --- docs/perf_counters.md | 3 +-- src/benchmark_runner.cc | 2 -- src/perf_counters.cc | 4 +++ src/perf_counters.h | 2 +- test/perf_counters_gtest.cc | 50 +++++++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/docs/perf_counters.md b/docs/perf_counters.md index 43ff451791..74560e9669 100644 --- a/docs/perf_counters.md +++ b/docs/perf_counters.md @@ -13,8 +13,7 @@ This feature is available if: Unit (PMU), * The benchmark is compiled with support for collecting counters. Currently, this requires [libpfm](http://perfmon2.sourceforge.net/) be available at build - time, and -* Currently, there is a limitation that the benchmark be run on one thread. + time The feature does not require modifying benchmark code. Counter collection is handled at the boundaries where timer collection is also handled. diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 7f3cd367a1..3cffbbfad5 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -163,8 +163,6 @@ class BenchmarkRunner { internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); - CHECK(b.threads() == 1 || !perf_counters_measurement.IsValid()) - << "Perf counters are not supported in multi-threaded cases.\n"; CHECK(FLAGS_benchmark_perf_counters.empty() || perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; diff --git a/src/perf_counters.cc b/src/perf_counters.cc index eb28cd99f8..4ddf0de250 100644 --- a/src/perf_counters.cc +++ b/src/perf_counters.cc @@ -67,6 +67,10 @@ PerfCounters PerfCounters::Create( return NoCounters(); } attr.disabled = is_first; + // Note: the man page for perf_event_create suggests inerit = true and + // read_format = PERF_FORMAT_GROUP don't work together, but that's not the + // case. + attr.inherit = true; attr.pinned = is_first; attr.exclude_kernel = true; attr.exclude_user = false; diff --git a/src/perf_counters.h b/src/perf_counters.h index 0c2c4616b3..b6629b9907 100644 --- a/src/perf_counters.h +++ b/src/perf_counters.h @@ -92,7 +92,7 @@ class PerfCounters final { // Take a snapshot of the current value of the counters into the provided // valid PerfCounterValues storage. The values are populated such that: // names()[i]'s value is (*values)[i] - BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) { + BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const { #ifndef BENCHMARK_OS_WINDOWS assert(values != nullptr); assert(IsValid()); diff --git a/test/perf_counters_gtest.cc b/test/perf_counters_gtest.cc index 47894af4f2..2a2868a715 100644 --- a/test/perf_counters_gtest.cc +++ b/test/perf_counters_gtest.cc @@ -1,3 +1,5 @@ +#include + #include "../src/perf_counters.h" #include "gtest/gtest.h" @@ -92,4 +94,52 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_GT(values2[0], 0); EXPECT_GT(values2[1], 0); } + +size_t do_work() { + size_t res = 0; + for (size_t i = 0; i < 100000000; ++i) res += i * i; + return res; +} + +void measure(size_t threadcount, PerfCounterValues* values1, + PerfCounterValues* values2) { + CHECK_NE(values1, nullptr); + CHECK_NE(values2, nullptr); + std::vector threads(threadcount); + auto work = [&]() { CHECK(do_work() > 1000); }; + + // We need to first set up the counters, then start the threads, so the + // threads would inherit the counters. But later, we need to first destroy the + // thread pool (so all the work finishes), then measure the counters. So the + // scopes overlap, and we need to explicitly control the scope of the + // threadpool. + auto counters = + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); + for (auto& t : threads) t = std::thread(work); + counters.Snapshot(values1); + for (auto& t : threads) t.join(); + counters.Snapshot(values2); +} + +TEST(PerfCountersTest, MultiThreaded) { + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported."; + } + EXPECT_TRUE(PerfCounters::Initialize()); + PerfCounterValues values1(2); + PerfCounterValues values2(2); + + measure(2, &values1, &values2); + std::vector D1{static_cast(values2[0] - values1[0]), + static_cast(values2[1] - values1[1])}; + + measure(4, &values1, &values2); + std::vector D2{static_cast(values2[0] - values1[0]), + static_cast(values2[1] - values1[1])}; + + // Some extra work will happen on the main thread - like joining the threads + // - so the ratio won't be quite 2.0, but very close. + EXPECT_GE(D2[0], 1.9 * D1[0]); + EXPECT_GE(D2[1], 1.9 * D1[1]); +} } // namespace From c983c3ecfb6e8cbb2a7e023a305ee22a43d5dbcc Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 20 May 2021 13:05:50 +0100 Subject: [PATCH 299/330] remove appveyor and add libera.chat as IRC resource --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a853115b36..a94925c223 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) -[![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) @@ -39,7 +38,9 @@ as some of the structural aspects of the APIs are similar. [Discussion group](https://groups.google.com/d/forum/benchmark-discuss) -IRC channel: [freenode](https://freenode.net) #googlebenchmark +IRC channels: +* [freenode](https://freenode.net) #googlebenchmark +* [libera](https://libera.chat) #benchmark [Additional Tooling Documentation](docs/tools.md) From a6a738c1cc3b754f1bccc0cefc3aeb551c341f88 Mon Sep 17 00:00:00 2001 From: haih-g Date: Thu, 20 May 2021 12:09:16 -0400 Subject: [PATCH 300/330] Implementation of random interleaving. (#1105) * Implementation of random interleaving. See http://github.com/google/benchmark/issues/1051 for the feature requests. Committer: Hai Huang (http://github.com/haih-g) On branch fr-1051 Changes to be committed: modified: include/benchmark/benchmark.h modified: src/benchmark.cc new file: src/benchmark_adjust_repetitions.cc new file: src/benchmark_adjust_repetitions.h modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: src/benchmark_register.cc modified: src/benchmark_runner.cc modified: src/benchmark_runner.h modified: test/CMakeLists.txt new file: test/benchmark_random_interleaving_gtest.cc * Fix benchmark_random_interleaving_gtest.cc for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc modified: src/benchmark_runner.cc modified: test/benchmark_random_interleaving_gtest.cc * Fix macos build for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: src/benchmark_runner.cc * Fix macos and windows build for fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_runner.cc * Fix benchmark_random_interleaving_test.cc for macos and windows in fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: test/benchmark_random_interleaving_gtest.cc * Fix int type benchmark_random_interleaving_gtest for macos in fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: test/benchmark_random_interleaving_gtest.cc * Address dominichamon's comments 03/29 for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: test/benchmark_random_interleaving_gtest.cc * Address dominichamon's comment on default min_time / repetitions for fr-1051. Also change sentinel of random_interleaving_repetitions to -1. Hopefully it fixes the failures on Windows. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h * Fix windows test failures for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_api_internal.cc modified: src/benchmark_runner.cc * Add license blurb for fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_adjust_repetitions.cc modified: src/benchmark_adjust_repetitions.h * Switch to std::shuffle() for fr-1105. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc * Change to 1e-9 in fr-1105 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_adjust_repetitions.cc * Fix broken build caused by bad merge for fr-1105. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_api_internal.cc modified: src/benchmark_runner.cc * Fix build breakage for fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: src/benchmark_register.cc modified: src/benchmark_runner.cc * Print out reports as they come in if random interleaving is disabled (fr-1051) Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc * size_t, int64_t --> int in benchmark_runner for fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_runner.cc modified: src/benchmark_runner.h * Address comments from dominichamon for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc modified: src/benchmark_adjust_repetitions.cc modified: src/benchmark_adjust_repetitions.h modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: test/benchmark_random_interleaving_gtest.cc * benchmar_indices --> size_t to make CI pass: fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark.cc * Fix min_time not initialized issue for fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h * min_time --> MinTime in fr-1051. Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: src/benchmark_api_internal.cc modified: src/benchmark_api_internal.h modified: src/benchmark_runner.cc * Add doc for random interleaving for fr-1051 Committer: Hai Huang On branch fr-1051 Your branch is up to date with 'origin/fr-1051'. Changes to be committed: modified: README.md new file: docs/random_interleaving.md Co-authored-by: Dominic Hamon --- README.md | 8 +- docs/random_interleaving.md | 26 ++ src/benchmark.cc | 212 +++++++++++---- src/benchmark_adjust_repetitions.cc | 125 +++++++++ src/benchmark_adjust_repetitions.h | 42 +++ src/benchmark_api_internal.cc | 34 ++- src/benchmark_api_internal.h | 30 ++- src/benchmark_runner.cc | 101 ++++++-- src/benchmark_runner.h | 7 +- test/CMakeLists.txt | 1 + test/benchmark_random_interleaving_gtest.cc | 271 ++++++++++++++++++++ 11 files changed, 772 insertions(+), 85 deletions(-) create mode 100644 docs/random_interleaving.md create mode 100644 src/benchmark_adjust_repetitions.cc create mode 100644 src/benchmark_adjust_repetitions.h create mode 100644 test/benchmark_random_interleaving_gtest.cc diff --git a/README.md b/README.md index a94925c223..19cf93036e 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ BENCHMARK_MAIN(); ``` To run the benchmark, compile and link against the `benchmark` library -(libbenchmark.a/.so). If you followed the build steps above, this library will +(libbenchmark.a/.so). If you followed the build steps above, this library will be under the build directory you created. ```bash @@ -300,6 +300,8 @@ too (`-lkstat`). [Setting the Time Unit](#setting-the-time-unit) +[Random Interleaving](docs/random_interleaving.md) + [User-Requested Performance Counters](docs/perf_counters.md) [Preventing Optimization](#preventing-optimization) @@ -400,8 +402,8 @@ Write benchmark results to a file with the `--benchmark_out=` option (or set `BENCHMARK_OUT`). Specify the output format with `--benchmark_out_format={json|console|csv}` (or set `BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is -deprecated and the saved `.csv` file -[is not parsable](https://github.com/google/benchmark/issues/794) by csv +deprecated and the saved `.csv` file +[is not parsable](https://github.com/google/benchmark/issues/794) by csv parsers. Specifying `--benchmark_out` does not suppress the console output. diff --git a/docs/random_interleaving.md b/docs/random_interleaving.md new file mode 100644 index 0000000000..2471b46bb0 --- /dev/null +++ b/docs/random_interleaving.md @@ -0,0 +1,26 @@ + + +# Random Interleaving + +[Random Interleaving](https://github.com/google/benchmark/issues/1051) is a +technique to lower run-to-run variance. It breaks the execution of a +microbenchmark into multiple chunks and randomly interleaves them with chunks +from other microbenchmarks in the same benchmark test. Data shows it is able to +lower run-to-run variance by +[40%](https://github.com/google/benchmark/issues/1051) on average. + +To use, set `--benchmark_enable_random_interleaving=true`. + +It's a known issue that random interleaving may increase the benchmark execution +time, if: + +1. A benchmark has costly setup and / or teardown. Random interleaving will run + setup and teardown many times and may increase test execution time + significantly. +2. The time to run a single benchmark iteration is larger than the desired time + per repetition (i.e., `benchmark_min_time / benchmark_repetitions`). + +The overhead of random interleaving can be controlled by +`--benchmark_random_interleaving_max_overhead`. The default value is 0.4 meaning +the total execution time under random interlaving is limited by 1.4 x original +total execution time. Set it to `inf` for unlimited overhead. diff --git a/src/benchmark.cc b/src/benchmark.cc index d205232b8a..272794e147 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -33,8 +33,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -54,6 +56,18 @@ #include "thread_manager.h" #include "thread_timer.h" +// Each benchmark can be repeated a number of times, and within each +// *repetition*, we run the user-defined benchmark function a number of +// *iterations*. The number of repetitions is determined based on flags +// (--benchmark_repetitions). +namespace { + +// Attempt to make each repetition run for at least this much of time. +constexpr double kDefaultMinTimeTotalSecs = 0.5; +constexpr int kRandomInterleavingDefaultRepetitions = 12; + +} // namespace + // Print a list of benchmarks. This option overrides all other options. DEFINE_bool(benchmark_list_tests, false); @@ -62,16 +76,39 @@ DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. DEFINE_string(benchmark_filter, "."); -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound -// on the total cpu time used by all threads that make up the test. For -// real-time based tests, this is the lower bound on the elapsed time of the -// benchmark execution, regardless of number of threads. -DEFINE_double(benchmark_min_time, 0.5); +// Do NOT read these flags directly. Use Get*() to read them. +namespace do_not_read_flag_directly { + +// Minimum number of seconds we should run benchmark per repetition before +// results are considered significant. For cpu-time based tests, this is the +// lower bound on the total cpu time used by all threads that make up the test. +// For real-time based tests, this is the lower bound on the elapsed time of the +// benchmark execution, regardless of number of threads. If left unset, will use +// kDefaultMinTimeTotalSecs / FLAGS_benchmark_repetitions, if random +// interleaving is enabled. Otherwise, will use kDefaultMinTimeTotalSecs. +// Do NOT read this flag directly. Use GetMinTime() to read this flag. +DEFINE_double(benchmark_min_time, -1.0); // The number of runs of each benchmark. If greater than 1, the mean and -// standard deviation of the runs will be reported. -DEFINE_int32(benchmark_repetitions, 1); +// standard deviation of the runs will be reported. By default, the number of +// repetitions is 1 if random interleaving is disabled, and up to +// kDefaultRepetitions if random interleaving is enabled. (Read the +// documentation for random interleaving to see why it might be less than +// kDefaultRepetitions.) +// Do NOT read this flag directly, Use GetRepetitions() to access this flag. +DEFINE_int32(benchmark_repetitions, -1); + +} // namespace do_not_read_flag_directly + +// The maximum overhead allowed for random interleaving. A value X means total +// execution time under random interleaving is limited by +// (1 + X) * original total execution time. Set to 'inf' to allow infinite +// overhead. +DEFINE_double(benchmark_random_interleaving_max_overhead, 0.4); + +// If set, enable random interleaving. See +// http://github.com/google/benchmark/issues/1051 for details. +DEFINE_bool(benchmark_enable_random_interleaving, false); // Report the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are reported for @@ -122,6 +159,30 @@ DEFINE_kvpairs(benchmark_context, {}); std::map* global_context = nullptr; +// Performance measurements always come with random variances. Defines a +// factor by which the required number of iterations is overestimated in order +// to reduce the probability that the minimum time requirement will not be met. +const double kSafetyMultiplier = 1.4; + +// Wraps --benchmark_min_time and returns valid default values if not supplied. +double GetMinTime() { + const double default_min_time = kDefaultMinTimeTotalSecs / GetRepetitions(); + const double flag_min_time = + do_not_read_flag_directly::FLAGS_benchmark_min_time; + return flag_min_time >= 0.0 ? flag_min_time : default_min_time; +} + +// Wraps --benchmark_repetitions and return valid default value if not supplied. +int GetRepetitions() { + const int default_repetitions = + FLAGS_benchmark_enable_random_interleaving + ? kRandomInterleavingDefaultRepetitions + : 1; + const int flag_repetitions = + do_not_read_flag_directly::FLAGS_benchmark_repetitions; + return flag_repetitions >= 0 ? flag_repetitions : default_repetitions; +} + // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} @@ -241,6 +302,39 @@ void State::FinishKeepRunning() { namespace internal { namespace { +// Flushes streams after invoking reporter methods that write to them. This +// ensures users get timely updates even when streams are not line-buffered. +void FlushStreams(BenchmarkReporter* reporter) { + if (!reporter) return; + std::flush(reporter->GetOutputStream()); + std::flush(reporter->GetErrorStream()); +}; + +// Reports in both display and file reporters. +void Report(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, const RunResults& run_results) { + auto report_one = [](BenchmarkReporter* reporter, + bool aggregates_only, + const RunResults& results) { + assert(reporter); + // If there are no aggregates, do output non-aggregates. + aggregates_only &= !results.aggregates_only.empty(); + if (!aggregates_only) + reporter->ReportRuns(results.non_aggregates); + if (!results.aggregates_only.empty()) + reporter->ReportRuns(results.aggregates_only); + }; + + report_one(display_reporter, run_results.display_report_aggregates_only, + run_results); + if (file_reporter) + report_one(file_reporter, run_results.file_report_aggregates_only, + run_results); + + FlushStreams(display_reporter); + FlushStreams(file_reporter); +}; + void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { @@ -248,7 +342,7 @@ void RunBenchmarks(const std::vector& benchmarks, CHECK(display_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. - bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; + bool might_have_aggregates = GetRepetitions() > 1; size_t name_field_width = 10; size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { @@ -256,8 +350,9 @@ void RunBenchmarks(const std::vector& benchmarks, std::max(name_field_width, benchmark.name().str().size()); might_have_aggregates |= benchmark.repetitions() > 1; - for (const auto& Stat : benchmark.statistics()) + for (const auto& Stat : benchmark.statistics()) { stat_field_width = std::max(stat_field_width, Stat.name_.size()); + } } if (might_have_aggregates) name_field_width += 1 + stat_field_width; @@ -268,45 +363,61 @@ void RunBenchmarks(const std::vector& benchmarks, // Keep track of running times of all instances of current benchmark std::vector complexity_reports; - // We flush streams after invoking reporter methods that write to them. This - // ensures users get timely updates even when streams are not line-buffered. - auto flushStreams = [](BenchmarkReporter* reporter) { - if (!reporter) return; - std::flush(reporter->GetOutputStream()); - std::flush(reporter->GetErrorStream()); - }; - if (display_reporter->ReportContext(context) && (!file_reporter || file_reporter->ReportContext(context))) { - flushStreams(display_reporter); - flushStreams(file_reporter); - - for (const auto& benchmark : benchmarks) { - RunResults run_results = RunBenchmark(benchmark, &complexity_reports); - - auto report = [&run_results](BenchmarkReporter* reporter, - bool report_aggregates_only) { - assert(reporter); - // If there are no aggregates, do output non-aggregates. - report_aggregates_only &= !run_results.aggregates_only.empty(); - if (!report_aggregates_only) - reporter->ReportRuns(run_results.non_aggregates); - if (!run_results.aggregates_only.empty()) - reporter->ReportRuns(run_results.aggregates_only); - }; - - report(display_reporter, run_results.display_report_aggregates_only); - if (file_reporter) - report(file_reporter, run_results.file_report_aggregates_only); - - flushStreams(display_reporter); - flushStreams(file_reporter); + FlushStreams(display_reporter); + FlushStreams(file_reporter); + + // Without random interleaving, benchmarks are executed in the order of: + // A, A, ..., A, B, B, ..., B, C, C, ..., C, ... + // That is, repetition is within RunBenchmark(), hence the name + // inner_repetitions. + // With random interleaving, benchmarks are executed in the order of: + // {Random order of A, B, C, ...}, {Random order of A, B, C, ...}, ... + // That is, repetitions is outside of RunBenchmark(), hence the name + // outer_repetitions. + int inner_repetitions = + FLAGS_benchmark_enable_random_interleaving ? 1 : GetRepetitions(); + int outer_repetitions = + FLAGS_benchmark_enable_random_interleaving ? GetRepetitions() : 1; + std::vector benchmark_indices(benchmarks.size()); + for (size_t i = 0; i < benchmarks.size(); ++i) { + benchmark_indices[i] = i; + } + + std::random_device rd; + std::mt19937 g(rd()); + // 'run_results_vector' and 'benchmarks' are parallel arrays. + std::vector run_results_vector(benchmarks.size()); + for (int i = 0; i < outer_repetitions; i++) { + if (FLAGS_benchmark_enable_random_interleaving) { + std::shuffle(benchmark_indices.begin(), benchmark_indices.end(), g); + } + for (size_t j : benchmark_indices) { + // Repetitions will be automatically adjusted under random interleaving. + if (!FLAGS_benchmark_enable_random_interleaving || + i < benchmarks[j].RandomInterleavingRepetitions()) { + RunBenchmark(benchmarks[j], outer_repetitions, inner_repetitions, + &complexity_reports, &run_results_vector[j]); + if (!FLAGS_benchmark_enable_random_interleaving) { + // Print out reports as they come in. + Report(display_reporter, file_reporter, run_results_vector.at(j)); + } + } + } + } + + if (FLAGS_benchmark_enable_random_interleaving) { + // Print out all reports at the end of the test. + for (const RunResults& run_results : run_results_vector) { + Report(display_reporter, file_reporter, run_results); + } } } display_reporter->Finalize(); if (file_reporter) file_reporter->Finalize(); - flushStreams(display_reporter); - flushStreams(file_reporter); + FlushStreams(display_reporter); + FlushStreams(file_reporter); } // Disable deprecated warnings temporarily because we need to reference @@ -456,6 +567,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n" " [--benchmark_format=]\n" @@ -476,10 +588,16 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag(argv[i], "benchmark_min_time", - &FLAGS_benchmark_min_time) || - ParseInt32Flag(argv[i], "benchmark_repetitions", - &FLAGS_benchmark_repetitions) || + ParseDoubleFlag( + argv[i], "benchmark_min_time", + &do_not_read_flag_directly::FLAGS_benchmark_min_time) || + ParseInt32Flag( + argv[i], "benchmark_repetitions", + &do_not_read_flag_directly::FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", + &FLAGS_benchmark_enable_random_interleaving) || + ParseDoubleFlag(argv[i], "benchmark_random_interleaving_max_overhead", + &FLAGS_benchmark_random_interleaving_max_overhead) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", &FLAGS_benchmark_report_aggregates_only) || ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", diff --git a/src/benchmark_adjust_repetitions.cc b/src/benchmark_adjust_repetitions.cc new file mode 100644 index 0000000000..2847927628 --- /dev/null +++ b/src/benchmark_adjust_repetitions.cc @@ -0,0 +1,125 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark_adjust_repetitions.h" + +#include "benchmark_api_internal.h" +#include "log.h" + +namespace benchmark { +namespace internal { + +namespace { + +constexpr double kNanosecondInSecond = 1e-9; + +} // namespace + +int ComputeRandomInterleavingRepetitions( + InternalRandomInterleavingRepetitionsInput input) { + // Find the repetitions such that total overhead is bounded. Let + // n = desired number of repetitions, i.e., the output of this method. + // t = total real execution time per repetition including overhead, + // (input.total_execution_time_per_repetition). + // o = maximum allowed increase in total real execution time due to random + // interleaving, measured as a fraction (input.max_overhead). + // e = estimated total execution time without Random Interleaving + // We want + // t * n / e <= 1 + o + // I.e., + // n <= (1 + o) * e / t + // + // Let + // h = overhead per repetition, which include all setup / teardown time and + // also the execution time of preliminary trials used to search for the + // correct number of iterations. + // r = real execution time per repetition not including overhead + // (input.real_accumulated_time_per_repetition). + // s = measured execution time per repetition not including overhead, + // which can be either real or CPU time + // (input.accumulated_time_per_repetition). + // We have + // h = t - r + // + // Let + // m = total minimum measured execution time for all repetitions + // (input.min_time_per_repetition * input.max_repetitions). + // Let + // f = m / s + // f is the scale factor between m and s, and will be used to estimate + // l, the total real execution time for all repetitions excluding the + // overhead. It's reasonable to assume that the real execution time excluding + // the overhead is proportional to the measured time. Hence we expect to see + // l / r to be equal to m / s. That is, l / r = f, thus, l = r * f. Then the + // total execution time e can be estimated by h + l, which is h + r * f. + // e = h + r * f + // Note that this might be an underestimation. If number of repetitions is + // reduced, we may need to run more iterations per repetition, and that may + // increase the number of preliminary trials needed to find the correct + // number of iterations. + + double h = std::max(0.0, input.total_execution_time_per_repetition - + input.real_time_used_per_repetition); + double r = + std::max(input.real_time_used_per_repetition, kNanosecondInSecond); + double s = + std::max(input.time_used_per_repetition, kNanosecondInSecond); + double m = input.min_time_per_repetition * input.max_repetitions; + + // f = m / s + // RunBenchmark() always overshoot the iteration count by kSafetyMultiplier. + // Apply the same factor here. + // f = kSafetyMultiplier * m / s + // Also we want to make sure 1 <= f <= input.max_repetitions. Note that we + // may not be able to reach m because the total iters per repetition is + // upper bounded by --benchmark_max_iters. This behavior is preserved in + // Random Interleaving, as we won't run repetitions more than + // input.max_repetitions to reach m. + + double f = kSafetyMultiplier * m / s; + f = std::min(std::max(f, 1.0), static_cast(input.max_repetitions)); + + double e = h + r * f; + // n <= (1 + o) * e / t = (1 + o) * e / (h + r) + // Also we want to make sure 1 <= n <= input.max_repetition, and (h + r) > 0. + double n = (1 + input.max_overhead) * e / (h + r); + n = std::min(std::max(n, 1.0), static_cast(input.max_repetitions)); + + int n_int = static_cast(n); + + VLOG(2) << "Computed random interleaving repetitions" + << "\n input.total_execution_time_per_repetition: " + << input.total_execution_time_per_repetition + << "\n input.time_used_per_repetition: " + << input.time_used_per_repetition + << "\n input.real_time_used_per_repetition: " + << input.real_time_used_per_repetition + << "\n input.min_time_per_repetitions: " + << input.min_time_per_repetition + << "\n input.max_repetitions: " << input.max_repetitions + << "\n input.max_overhead: " << input.max_overhead + << "\n h: " << h + << "\n r: " << r + << "\n s: " << s + << "\n f: " << f + << "\n m: " << m + << "\n e: " << e + << "\n n: " << n + << "\n n_int: " << n_int; + + return n_int; +} + +} // internal +} // benchmark diff --git a/src/benchmark_adjust_repetitions.h b/src/benchmark_adjust_repetitions.h new file mode 100644 index 0000000000..21a666afe0 --- /dev/null +++ b/src/benchmark_adjust_repetitions.h @@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_ADJUST_REPETITIONS_H +#define BENCHMARK_ADJUST_REPETITIONS_H + +#include "benchmark/benchmark.h" +#include "commandlineflags.h" + +namespace benchmark { +namespace internal { + +// Defines the input tuple to ComputeRandomInterleavingRepetitions(). +struct InternalRandomInterleavingRepetitionsInput { + double total_execution_time_per_repetition; + double time_used_per_repetition; + double real_time_used_per_repetition; + double min_time_per_repetition; + double max_overhead; + int max_repetitions; +}; + +// Should be called right after the first repetition is completed to estimate +// the number of iterations. +int ComputeRandomInterleavingRepetitions( + InternalRandomInterleavingRepetitionsInput input); + +} // end namespace internal +} // end namespace benchmark + +#endif // BENCHMARK_ADJUST_REPETITIONS_H diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 553ff44e5c..ddd46bee63 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -2,8 +2,11 @@ #include +#include "check.h" #include "string_util.h" +DECLARE_bool(benchmark_enable_random_interleaving); + namespace benchmark { namespace internal { @@ -21,9 +24,12 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, complexity_lambda_(benchmark_.complexity_lambda_), statistics_(benchmark_.statistics_), repetitions_(benchmark_.repetitions_), - min_time_(benchmark_.min_time_), + min_time_(!IsZero(benchmark_.min_time_) ? benchmark_.min_time_ + : GetMinTime()), iterations_(benchmark_.iterations_), threads_(thread_count) { + CHECK(!IsZero(min_time_)) << "min_time must be non-zero."; + name_.function_name = benchmark_.name_; size_t arg_i = 0; @@ -77,6 +83,32 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, } } +double BenchmarkInstance::MinTime() const { + if (FLAGS_benchmark_enable_random_interleaving) { + // Random Interleaving will automatically adjust + // random_interleaving_repetitions(). Dividing + // total execution time by random_interleaving_repetitions() gives + // the adjusted min_time per repetition. + return min_time_ * GetRepetitions() / RandomInterleavingRepetitions(); + } + return min_time_; +} + +int BenchmarkInstance::RandomInterleavingRepetitions() const { + return random_interleaving_repetitions_ < 0 + ? GetRepetitions() + : random_interleaving_repetitions_; +} + +bool BenchmarkInstance::RandomInterleavingRepetitionsInitialized() const { + return random_interleaving_repetitions_ >= 0; +} + +void BenchmarkInstance::InitRandomInterleavingRepetitions( + int reps) const { + random_interleaving_repetitions_ = reps; +} + State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index b0595e5a12..0ff8dafbe6 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -1,6 +1,10 @@ #ifndef BENCHMARK_API_INTERNAL_H #define BENCHMARK_API_INTERNAL_H +#include "benchmark/benchmark.h" +#include "commandlineflags.h" + +#include #include #include #include @@ -14,12 +18,25 @@ namespace benchmark { namespace internal { +extern const double kSafetyMultiplier; + // Information kept per benchmark we may want to run class BenchmarkInstance { public: BenchmarkInstance(Benchmark* benchmark, const std::vector& args, int threads); + // Returns number of repetitions for Random Interleaving. This will be + // initialized later once we finish the first repetition, if Random + // Interleaving is enabled. See also ComputeRandominterleavingrepetitions(). + int RandomInterleavingRepetitions() const; + + // Returns true if repetitions for Random Interleaving is initialized. + bool RandomInterleavingRepetitionsInitialized() const; + + // Initializes number of repetitions for random interleaving. + void InitRandomInterleavingRepetitions(int reps) const; + const BenchmarkName& name() const { return name_; } AggregationReportMode aggregation_report_mode() const { return aggregation_report_mode_; @@ -32,7 +49,7 @@ class BenchmarkInstance { BigOFunc& complexity_lambda() const { return *complexity_lambda_; } const std::vector& statistics() const { return statistics_; } int repetitions() const { return repetitions_; } - double min_time() const { return min_time_; } + double MinTime() const; IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } @@ -53,12 +70,13 @@ class BenchmarkInstance { bool use_manual_time_; BigO complexity_; BigOFunc* complexity_lambda_; - UserCounters counters_; - const std::vector& statistics_; + std::vector statistics_; int repetitions_; double min_time_; IterationCount iterations_; - int threads_; // Number of concurrent threads to us + int threads_; + UserCounters counters_; + mutable int random_interleaving_repetitions_ = -1; }; bool FindBenchmarksInternal(const std::string& re, @@ -69,6 +87,10 @@ bool IsZero(double n); ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); +double GetMinTime(); + +int GetRepetitions(); + } // end namespace internal } // end namespace benchmark diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 3cffbbfad5..330cb4494b 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -15,6 +15,7 @@ #include "benchmark_runner.h" #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" +#include "benchmark_adjust_repetitions.h" #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS @@ -52,6 +53,9 @@ #include "thread_manager.h" #include "thread_timer.h" +DECLARE_bool(benchmark_enable_random_interleaving); +DECLARE_double(benchmark_random_interleaving_max_overhead); + namespace benchmark { namespace internal { @@ -67,7 +71,7 @@ BenchmarkReporter::Run CreateRunReport( const internal::ThreadManager::Result& results, IterationCount memory_iterations, const MemoryManager::Result& memory_result, double seconds, - int64_t repetition_index) { + int repetition_index) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -138,12 +142,16 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, - std::vector* complexity_reports_) + int outer_repetitions_, + int inner_repetitions_, + std::vector* complexity_reports_, + RunResults* run_results_) : b(b_), complexity_reports(*complexity_reports_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), - repeats(b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), + run_results(*run_results_), + outer_repetitions(outer_repetitions_), + inner_repetitions(inner_repetitions_), + repeats(b.repetitions() != 0 ? b.repetitions() : inner_repetitions), has_explicit_iteration_count(b.iterations() != 0), pool(b.threads() - 1), iters(has_explicit_iteration_count ? b.iterations() : 1), @@ -163,6 +171,7 @@ class BenchmarkRunner { internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); + CHECK(FLAGS_benchmark_perf_counters.empty() || perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; @@ -179,21 +188,20 @@ class BenchmarkRunner { if ((b.complexity() != oNone) && b.last_benchmark_instance) { auto additional_run_stats = ComputeBigO(complexity_reports); run_results.aggregates_only.insert(run_results.aggregates_only.end(), - additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.begin(), + additional_run_stats.end()); complexity_reports.clear(); } } - RunResults&& get_results() { return std::move(run_results); } - private: - RunResults run_results; - const benchmark::internal::BenchmarkInstance& b; std::vector& complexity_reports; - const double min_time; + RunResults& run_results; + + const int outer_repetitions; + const int inner_repetitions; const int repeats; const bool has_explicit_iteration_count; @@ -268,13 +276,14 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); + double multiplier = + b.MinTime() * kSafetyMultiplier / std::max(i.seconds, 1e-9); // If our last run was at least 10% of FLAGS_benchmark_min_time then we // use the multiplier directly. // Otherwise we use at most 10 times expansion. // NOTE: When the last run was at least 10% of the min time the max // expansion should be 14x. - bool is_significant = (i.seconds / min_time) > 0.1; + bool is_significant = (i.seconds / b.MinTime()) > 0.1; multiplier = is_significant ? multiplier : std::min(10.0, multiplier); if (multiplier <= 1.0) multiplier = 2.0; @@ -295,14 +304,17 @@ class BenchmarkRunner { // or because an error was reported. return i.results.has_error_ || i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= min_time || // The elapsed time is large enough. - // CPU time is specified but the elapsed real time greatly exceeds - // the minimum time. - // Note that user provided timers are except from this sanity check. - ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); + i.seconds >= b.MinTime() || // The elapsed time is large enough. + // CPU time is specified but the + // elapsed real time greatly exceeds + // the minimum time. Note that user + // provided timers are except from this + // sanity check. + ((i.results.real_time_used >= 5 * b.MinTime()) && + !b.use_manual_time()); } - void DoOneRepetition(int64_t repetition_index) { + void DoOneRepetition(int repetition_index) { const bool is_the_first_repetition = repetition_index == 0; IterationResults i; @@ -312,8 +324,10 @@ class BenchmarkRunner { // Please do note that the if there are repetitions, the iteration count // is *only* calculated for the *first* repetition, and other repetitions // simply use that precomputed iteration count. + const auto exec_start = benchmark::ChronoClockNow(); for (;;) { i = DoNIterations(); + const auto exec_end = benchmark::ChronoClockNow(); // Do we consider the results to be significant? // If we are doing repetitions, and the first repetition was already done, @@ -324,7 +338,38 @@ class BenchmarkRunner { has_explicit_iteration_count || ShouldReportIterationResults(i); - if (results_are_significant) break; // Good, let's report them! + if (results_are_significant) { + // The number of repetitions for random interleaving may be reduced + // to limit the increase in benchmark execution time. When this happens + // the target execution time for each repetition is increased. We may + // need to rerun trials to calculate iters according to the increased + // target execution time. + bool rerun_trial = false; + // If random interleaving is enabled and the repetitions is not + // initialized, do it now. + if (FLAGS_benchmark_enable_random_interleaving && + !b.RandomInterleavingRepetitionsInitialized()) { + InternalRandomInterleavingRepetitionsInput input; + input.total_execution_time_per_repetition = exec_end - exec_start; + input.time_used_per_repetition = i.seconds; + input.real_time_used_per_repetition = i.results.real_time_used; + input.min_time_per_repetition = GetMinTime(); + input.max_overhead = FLAGS_benchmark_random_interleaving_max_overhead; + input.max_repetitions = GetRepetitions(); + b.InitRandomInterleavingRepetitions( + ComputeRandomInterleavingRepetitions(input)); + // If the number of repetitions changed, need to rerun the last trial + // because iters may also change. Note that we only need to do this + // if accumulated_time < b.MinTime(), i.e., the iterations we have + // run is not enough for the already adjusted b.MinTime(). + // Otherwise, we will still skip the rerun. + rerun_trial = + b.RandomInterleavingRepetitions() < GetRepetitions() && + i.seconds < b.MinTime() && !has_explicit_iteration_count; + } + + if (!rerun_trial) break; // Good, let's report them! + } // Nope, bad iteration. Let's re-estimate the hopefully-sufficient // iteration count, and run the benchmark again... @@ -341,7 +386,8 @@ class BenchmarkRunner { if (memory_manager != nullptr) { // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. - memory_iterations = std::min(16, iters); + memory_iterations = std::min( + 16 / outer_repetitions + (16 % outer_repetitions != 0), iters); memory_manager->Start(); std::unique_ptr manager; manager.reset(new internal::ThreadManager(1)); @@ -367,11 +413,12 @@ class BenchmarkRunner { } // end namespace -RunResults RunBenchmark( - const benchmark::internal::BenchmarkInstance& b, - std::vector* complexity_reports) { - internal::BenchmarkRunner r(b, complexity_reports); - return r.get_results(); +void RunBenchmark(const benchmark::internal::BenchmarkInstance& b, + const int outer_repetitions, const int inner_repetitions, + std::vector* complexity_reports, + RunResults* run_results) { + internal::BenchmarkRunner r(b, outer_repetitions, inner_repetitions, + complexity_reports, run_results); } } // end namespace internal diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 9b0cf2a64e..e29aa32306 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -42,9 +42,10 @@ struct RunResults { bool file_report_aggregates_only = false; }; -RunResults RunBenchmark( - const benchmark::internal::BenchmarkInstance& b, - std::vector* complexity_reports); +void RunBenchmark(const benchmark::internal::BenchmarkInstance& b, + int outer_repetitions, int inner_repetitions, + std::vector* complexity_reports, + RunResults* run_results); } // namespace internal diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1e7b6829f2..8945f5005f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -196,6 +196,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(benchmark_gtest) add_gtest(benchmark_name_gtest) + add_gtest(benchmark_random_interleaving_gtest) add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc new file mode 100644 index 0000000000..5e8329a4e6 --- /dev/null +++ b/test/benchmark_random_interleaving_gtest.cc @@ -0,0 +1,271 @@ +#include +#include +#include + +#include "../src/benchmark_adjust_repetitions.h" +#include "../src/string_util.h" +#include "benchmark/benchmark.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +DECLARE_bool(benchmark_enable_random_interleaving); +DECLARE_string(benchmark_filter); +DECLARE_double(benchmark_random_interleaving_max_overhead); + +namespace do_not_read_flag_directly { +DECLARE_int32(benchmark_repetitions); +} // namespace do_not_read_flag_directly + +namespace benchmark { +namespace internal { +namespace { + +class EventQueue : public std::queue { + public: + void Put(const std::string& event) { + push(event); + } + + void Clear() { + while (!empty()) { + pop(); + } + } + + std::string Get() { + std::string event = front(); + pop(); + return event; + } +}; + +static EventQueue* queue = new EventQueue; + +class NullReporter : public BenchmarkReporter { + public: + bool ReportContext(const Context& /*context*/) override { + return true; + } + void ReportRuns(const std::vector& /* report */) override {} +}; + +class BenchmarkTest : public testing::Test { + public: + static void SetupHook(int /* num_threads */) { queue->push("Setup"); } + + static void TeardownHook(int /* num_threads */) { queue->push("Teardown"); } + + void Execute(const std::string& pattern) { + queue->Clear(); + + BenchmarkReporter* reporter = new NullReporter; + FLAGS_benchmark_filter = pattern; + RunSpecifiedBenchmarks(reporter); + delete reporter; + + queue->Put("DONE"); // End marker + } +}; + +static void BM_Match1(benchmark::State& state) { + const int64_t arg = state.range(0); + + for (auto _ : state) {} + queue->Put(StrFormat("BM_Match1/%d", static_cast(arg))); +} +BENCHMARK(BM_Match1) + ->Iterations(100) + ->Arg(1) + ->Arg(2) + ->Arg(3) + ->Range(10, 80) + ->Args({90}) + ->Args({100}); + +static void BM_MatchOverhead(benchmark::State& state) { + const int64_t arg = state.range(0); + + for (auto _ : state) {} + queue->Put(StrFormat("BM_MatchOverhead/%d", static_cast(arg))); +} +BENCHMARK(BM_MatchOverhead) + ->Iterations(100) + ->Arg(64) + ->Arg(80); + +TEST_F(BenchmarkTest, Match1) { + Execute("BM_Match1"); + ASSERT_EQ("BM_Match1/1", queue->Get()); + ASSERT_EQ("BM_Match1/2", queue->Get()); + ASSERT_EQ("BM_Match1/3", queue->Get()); + ASSERT_EQ("BM_Match1/10", queue->Get()); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("BM_Match1/90", queue->Get()); + ASSERT_EQ("BM_Match1/100", queue->Get()); + ASSERT_EQ("DONE", queue->Get()); +} + +TEST_F(BenchmarkTest, Match1WithRepetition) { + do_not_read_flag_directly::FLAGS_benchmark_repetitions = 2; + + Execute("BM_Match1/(64|80)"); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("DONE", queue->Get()); +} + +TEST_F(BenchmarkTest, Match1WithRandomInterleaving) { + FLAGS_benchmark_enable_random_interleaving = true; + do_not_read_flag_directly::FLAGS_benchmark_repetitions = 100; + FLAGS_benchmark_random_interleaving_max_overhead = + std::numeric_limits::infinity(); + + std::vector expected({"BM_Match1/64", "BM_Match1/80"}); + std::map interleaving_count; + Execute("BM_Match1/(64|80)"); + for (int i = 0; i < 100; ++i) { + std::vector interleaving; + interleaving.push_back(queue->Get()); + interleaving.push_back(queue->Get()); + EXPECT_THAT(interleaving, testing::UnorderedElementsAreArray(expected)); + interleaving_count[StrFormat("%s,%s", interleaving[0].c_str(), + interleaving[1].c_str())]++; + } + EXPECT_GE(interleaving_count.size(), 2) << "Interleaving was not randomized."; + ASSERT_EQ("DONE", queue->Get()); +} + +TEST_F(BenchmarkTest, Match1WithRandomInterleavingAndZeroOverhead) { + FLAGS_benchmark_enable_random_interleaving = true; + do_not_read_flag_directly::FLAGS_benchmark_repetitions = 100; + FLAGS_benchmark_random_interleaving_max_overhead = 0; + + // ComputeRandomInterleavingRepetitions() will kick in and rerun each + // benchmark once with increased iterations. Then number of repetitions will + // be reduced to < 100. The first 4 executions should be + // 2 x BM_MatchOverhead/64 and 2 x BM_MatchOverhead/80. + std::vector expected( + {"BM_MatchOverhead/64", "BM_MatchOverhead/80", "BM_MatchOverhead/64", + "BM_MatchOverhead/80"}); + std::map interleaving_count; + Execute("BM_MatchOverhead/(64|80)"); + std::vector interleaving; + interleaving.push_back(queue->Get()); + interleaving.push_back(queue->Get()); + interleaving.push_back(queue->Get()); + interleaving.push_back(queue->Get()); + EXPECT_THAT(interleaving, testing::UnorderedElementsAreArray(expected)); + ASSERT_LT(queue->size(), 100) << "# Repetitions was not reduced to < 100."; +} + +InternalRandomInterleavingRepetitionsInput CreateInput( + double total, double time, double real_time, double min_time, + double overhead, int repetitions) { + InternalRandomInterleavingRepetitionsInput input; + input.total_execution_time_per_repetition = total; + input.time_used_per_repetition = time; + input.real_time_used_per_repetition = real_time; + input.min_time_per_repetition = min_time; + input.max_overhead = overhead; + input.max_repetitions = repetitions; + return input; +} + +TEST(Benchmark, ComputeRandomInterleavingRepetitions) { + // On wall clock time. + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.05, 0.05, 0.05, 0.05, 0.0, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.05, 0.05, 0.05, 0.05, 0.4, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.06, 0.05, 0.05, 0.05, 0.0, 10)), + 8); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.06, 0.05, 0.05, 0.05, 0.4, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.08, 0.05, 0.05, 0.05, 0.0, 10)), + 6); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.08, 0.05, 0.05, 0.05, 0.4, 10)), + 9); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.26, 0.25, 0.25, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.25, 0.25, 0.25, 0.05, 0.4, 10)), + 3); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.26, 0.25, 0.25, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.26, 0.25, 0.25, 0.05, 0.4, 10)), + 3); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.38, 0.25, 0.25, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.38, 0.25, 0.25, 0.05, 0.4, 10)), + 3); + + // On CPU time. + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.1, 0.05, 0.1, 0.05, 0.0, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.1, 0.05, 0.1, 0.05, 0.4, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.11, 0.05, 0.1, 0.05, 0.0, 10)), + 9); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.11, 0.05, 0.1, 0.05, 0.4, 10)), + 10); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.15, 0.05, 0.1, 0.05, 0.0, 10)), + 7); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.15, 0.05, 0.1, 0.05, 0.4, 10)), + 9); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.5, 0.25, 0.5, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.5, 0.25, 0.5, 0.05, 0.4, 10)), + 3); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.51, 0.25, 0.5, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.51, 0.25, 0.5, 0.05, 0.4, 10)), + 3); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.8, 0.25, 0.5, 0.05, 0.0, 10)), + 2); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.8, 0.25, 0.5, 0.05, 0.4, 10)), + 2); + + // Corner cases. + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.0, 0.25, 0.5, 0.05, 0.4, 10)), + 3); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.8, 0.0, 0.5, 0.05, 0.4, 10)), + 9); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.8, 0.25, 0.0, 0.05, 0.4, 10)), + 1); + EXPECT_EQ(ComputeRandomInterleavingRepetitions( + CreateInput(0.8, 0.25, 0.5, 0.0, 0.4, 10)), + 1); +} + +} // namespace +} // namespace internal +} // namespace benchmark From a4bcd937b298fdc2b0c7da9961fa202a5aecd56b Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 20 May 2021 12:59:29 -0400 Subject: [PATCH 301/330] fix version recorded in releases (#1047) * cmake: fix handling the case where `git describe` fails * cmake: fix version recorded in releases If downloaded as a tarball release, there will be no info from git to determine the release, so it ends up v0.0.0. If that's the case, we'll now use the release specified in the project() command, which needs to be updated for each new release. * cmake: add `--tags` to `git describe` That way, lightweight tags will also be taken into account, which should never hurt, but it'll help in cases where, for some mysterious reason or other, annotated tags don't make it into a clone. * update releasing.md --- CMakeLists.txt | 10 ++++++++-- cmake/GetGitVersion.cmake | 22 +++++++++++++--------- docs/releasing.md | 6 ++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e75d41f4f6..5a0e17c3de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ foreach(p endif() endforeach() -project (benchmark CXX) +project (benchmark VERSION 1.5.3 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -94,8 +94,14 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(GetGitVersion) get_git_version(GIT_VERSION) +# If no git version can be determined, use the version +# from the project() command +if ("${GIT_VERSION}" STREQUAL "0.0.0") + set(VERSION "${benchmark_VERSION}") +else() + set(VERSION "${GIT_VERSION}") +endif() # Tell the user what versions we are using -string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" VERSION ${GIT_VERSION}) message(STATUS "Version: ${VERSION}") # The version of the libraries diff --git a/cmake/GetGitVersion.cmake b/cmake/GetGitVersion.cmake index 4f10f226d7..04a1f9b70d 100644 --- a/cmake/GetGitVersion.cmake +++ b/cmake/GetGitVersion.cmake @@ -20,16 +20,20 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status - OUTPUT_VARIABLE GIT_VERSION + OUTPUT_VARIABLE GIT_DESCRIBE_VERSION ERROR_QUIET) - if(${status}) - set(GIT_VERSION "v0.0.0") + if(status) + set(GIT_DESCRIBE_VERSION "v0.0.0") + endif() + + string(STRIP ${GIT_DESCRIBE_VERSION} GIT_DESCRIBE_VERSION) + if(GIT_DESCRIBE_VERSION MATCHES v[^-]*-) + string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" GIT_VERSION ${GIT_DESCRIBE_VERSION}) else() - string(STRIP ${GIT_VERSION} GIT_VERSION) - string(REGEX REPLACE "-[0-9]+-g" "-" GIT_VERSION ${GIT_VERSION}) + string(REGEX REPLACE "v(.*)" "\\1" GIT_VERSION ${GIT_DESCRIBE_VERSION}) endif() # Work out if the repository is dirty @@ -43,12 +47,12 @@ function(get_git_version var) ERROR_QUIET) string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) if (${GIT_DIRTY}) - set(GIT_VERSION "${GIT_VERSION}-dirty") + set(GIT_DESCRIBE_VERSION "${GIT_DESCRIBE_VERSION}-dirty") endif() + message(STATUS "git version: ${GIT_DESCRIBE_VERSION} normalized to ${GIT_VERSION}") else() - set(GIT_VERSION "v0.0.0") + set(GIT_VERSION "0.0.0") endif() - message(STATUS "git Version: ${GIT_VERSION}") set(${var} ${GIT_VERSION} PARENT_SCOPE) endfunction() diff --git a/docs/releasing.md b/docs/releasing.md index f0cd7010e3..9e16ff2dd2 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,6 +8,12 @@ * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of commits between the last annotated tag and HEAD * Pick the most interesting. +* Create one last commit that updates the version saved in `CMakeLists.txt` to the release version you're creating. (This version will be used if benchmark is installed from the archive you'll be creating in the next step.) + +``` +project (benchmark VERSION 1.5.3 LANGUAGES CXX) +``` + * Create a release through github's interface * Note this will create a lightweight tag. * Update this to an annotated tag: From db2de74cc8c34131a6f673e35751935cc1897a0d Mon Sep 17 00:00:00 2001 From: Mariusz Wachowicz Date: Fri, 21 May 2021 10:48:20 +0200 Subject: [PATCH 302/330] Fix pedantic compilation flag violation (#1156) ';' after method definition was removed. Also, pedantic flag is now uncommented in CMakeList.txt. --- CMakeLists.txt | 5 ++--- src/benchmark.cc | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a0e17c3de..53c28932ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,9 +167,8 @@ else() # Disable warning when compiling tests as gtest does not use 'override'. add_cxx_compiler_flag(-Wsuggest-override) endif() - # Disabled until googletest (gmock) stops emitting variadic macro warnings - #add_cxx_compiler_flag(-pedantic) - #add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) add_cxx_compiler_flag(-Wshorten-64-to-32) add_cxx_compiler_flag(-fstrict-aliasing) # Disable warnings regarding deprecated parts of the library while building diff --git a/src/benchmark.cc b/src/benchmark.cc index 272794e147..97d657a08f 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -308,7 +308,7 @@ void FlushStreams(BenchmarkReporter* reporter) { if (!reporter) return; std::flush(reporter->GetOutputStream()); std::flush(reporter->GetErrorStream()); -}; +} // Reports in both display and file reporters. void Report(BenchmarkReporter* display_reporter, @@ -333,7 +333,7 @@ void Report(BenchmarkReporter* display_reporter, FlushStreams(display_reporter); FlushStreams(file_reporter); -}; +} void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, From 0e1255af2fdad14e688a97450391e52b35a7a61f Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 28 May 2021 11:09:18 +0100 Subject: [PATCH 303/330] Removing freenode from README It seems that by setting the /topic in freenode #googlebenchmark to point to libera I have angered the powers that be and we've been locked out of the channel. Libera it is then. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 19cf93036e..6e5844b010 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ as some of the structural aspects of the APIs are similar. [Discussion group](https://groups.google.com/d/forum/benchmark-discuss) IRC channels: -* [freenode](https://freenode.net) #googlebenchmark * [libera](https://libera.chat) #benchmark [Additional Tooling Documentation](docs/tools.md) From bc5651e54a7e178ca6b1e27e469a9be19cfa62c4 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Sun, 30 May 2021 09:48:05 +0100 Subject: [PATCH 304/330] bump version to v1.5.4 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53c28932ee..ef8dcdc68c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ foreach(p endif() endforeach() -project (benchmark VERSION 1.5.3 LANGUAGES CXX) +project (benchmark VERSION 1.5.4 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) From 09a87e319454fac2681caac0d4fe98cb6ec23cf5 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Sun, 30 May 2021 09:58:57 +0100 Subject: [PATCH 305/330] Set theme jekyll-theme-hacker --- docs/_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_config.yml b/docs/_config.yml index 18854876c6..fc24e7a62d 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1 +1 @@ -theme: jekyll-theme-midnight \ No newline at end of file +theme: jekyll-theme-hacker \ No newline at end of file From 604112c2d58c90d49280637c262cdcb7406fb311 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 1 Jun 2021 11:00:37 +0100 Subject: [PATCH 306/330] Run build-and-test on all branches --- .github/workflows/build-and-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 955da5f9b7..9e5be3b1dc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,10 +1,8 @@ name: build-and-test on: - push: - branches: [master] - pull_request: - branches: [master] + push: {} + pull_request: {} jobs: # TODO: add 32-bit builds (g++ and clang++) for ubuntu From f10b9c0cb3e982bbe6247e080b99ab31e8a67970 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Tue, 1 Jun 2021 13:01:49 +0300 Subject: [PATCH 307/330] Un-disable github actions :] --- .github/workflows/bazel.yml | 2 +- .github/workflows/build-and-test-perfcounters.yml | 4 ++-- .github/workflows/build-and-test.yml | 6 ++++-- .github/workflows/pylint.yml | 4 ++-- .github/workflows/test_bindings.yml | 4 ++-- docs/releasing.md | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 99f7e26054..a53661b2f9 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -19,7 +19,7 @@ jobs: path: "~/.cache/bazel" key: ${{ env.cache-name }}-${{ runner.os }}-${{ github.ref }} restore-keys: | - ${{ env.cache-name }}-${{ runner.os }}-master + ${{ env.cache-name }}-${{ runner.os }}-main - name: build run: | diff --git a/.github/workflows/build-and-test-perfcounters.yml b/.github/workflows/build-and-test-perfcounters.yml index dfb88cbc3e..b2b5419197 100644 --- a/.github/workflows/build-and-test-perfcounters.yml +++ b/.github/workflows/build-and-test-perfcounters.yml @@ -2,9 +2,9 @@ name: build-and-test-perfcounters on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: job: diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9e5be3b1dc..ab67de9e96 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,8 +1,10 @@ name: build-and-test on: - push: {} - pull_request: {} + push: + branches: [main] + pull_request: + branches: [main] jobs: # TODO: add 32-bit builds (g++ and clang++) for ubuntu diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c8696749f3..0f73a58232 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -2,9 +2,9 @@ name: pylint on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: pylint: diff --git a/.github/workflows/test_bindings.yml b/.github/workflows/test_bindings.yml index 273d7f93ee..4a580ebe04 100644 --- a/.github/workflows/test_bindings.yml +++ b/.github/workflows/test_bindings.yml @@ -2,9 +2,9 @@ name: test-bindings on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] jobs: python_bindings: diff --git a/docs/releasing.md b/docs/releasing.md index 9e16ff2dd2..7a6dfc4017 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,6 +1,6 @@ # How to release -* Make sure you're on master and synced to HEAD +* Make sure you're on main and synced to HEAD * Ensure the project builds and tests run (sanity check only, obviously) * `parallel -j0 exec ::: test/*_test` can help ensure everything at least passes From 4ff734960c5f1eae10ecc4df36b2a2e33c326ce4 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 1 Jun 2021 15:48:44 +0100 Subject: [PATCH 308/330] Run build-and-test on all branches --- .github/workflows/build-and-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ab67de9e96..9e5be3b1dc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,10 +1,8 @@ name: build-and-test on: - push: - branches: [main] - pull_request: - branches: [main] + push: {} + pull_request: {} jobs: # TODO: add 32-bit builds (g++ and clang++) for ubuntu From 6f094ba13e44a4725d25d36c3fdfa5e309879546 Mon Sep 17 00:00:00 2001 From: Norman Heino Date: Tue, 1 Jun 2021 16:50:42 +0200 Subject: [PATCH 309/330] Fix perf counter argument parsing (#1160) * Fix argument order in StrSplit * Update AUTHORS, CONTRIBUTORS --- AUTHORS | 1 + CONTRIBUTORS | 1 + src/string_util.cc | 2 +- test/string_util_gtest.cc | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9b980419f2..838dd4f5bd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -43,6 +43,7 @@ Matt Clarkson Maxim Vafin MongoDB Inc. Nick Hutchinson +Norman Heino Oleksandr Sochka Ori Livneh Paul Redmond diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 8370a2d737..7489731de5 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -62,6 +62,7 @@ Lei Xu Matt Clarkson Maxim Vafin Nick Hutchinson +Norman Heino Oleksandr Sochka Ori Livneh Pascal Leroy diff --git a/src/string_util.cc b/src/string_util.cc index cb973a7c6f..3551418174 100644 --- a/src/string_util.cc +++ b/src/string_util.cc @@ -169,7 +169,7 @@ std::vector StrSplit(const std::string& str, char delim) { size_t first = 0; size_t next = str.find(delim); for (; next != std::string::npos; - first = next + 1, next = str.find(first, delim)) { + first = next + 1, next = str.find(delim, first)) { ret.push_back(str.substr(first, next - first)); } ret.push_back(str.substr(first)); diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc index 1ad9fb9d26..c7061b409e 100644 --- a/test/string_util_gtest.cc +++ b/test/string_util_gtest.cc @@ -154,8 +154,8 @@ TEST(StringUtilTest, StrSplit) { EXPECT_EQ(benchmark::StrSplit("", ','), std::vector{}); EXPECT_EQ(benchmark::StrSplit("hello", ','), std::vector({"hello"})); - EXPECT_EQ(benchmark::StrSplit("hello,there", ','), - std::vector({"hello", "there"})); + EXPECT_EQ(benchmark::StrSplit("hello,there,is,more", ','), + std::vector({"hello", "there", "is", "more"})); } } // end namespace From e025dd5a54d8cefda429964ad3d2c7f4994ed38e Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Tue, 1 Jun 2021 16:05:50 +0100 Subject: [PATCH 310/330] Revert "Implementation of random interleaving. (#1105)" (#1161) This reverts commit a6a738c1cc3b754f1bccc0cefc3aeb551c341f88. --- README.md | 8 +- docs/random_interleaving.md | 26 -- src/benchmark.cc | 212 ++++----------- src/benchmark_adjust_repetitions.cc | 125 --------- src/benchmark_adjust_repetitions.h | 42 --- src/benchmark_api_internal.cc | 34 +-- src/benchmark_api_internal.h | 30 +-- src/benchmark_runner.cc | 101 ++------ src/benchmark_runner.h | 7 +- test/CMakeLists.txt | 1 - test/benchmark_random_interleaving_gtest.cc | 271 -------------------- 11 files changed, 85 insertions(+), 772 deletions(-) delete mode 100644 docs/random_interleaving.md delete mode 100644 src/benchmark_adjust_repetitions.cc delete mode 100644 src/benchmark_adjust_repetitions.h delete mode 100644 test/benchmark_random_interleaving_gtest.cc diff --git a/README.md b/README.md index 6e5844b010..41cada9553 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ BENCHMARK_MAIN(); ``` To run the benchmark, compile and link against the `benchmark` library -(libbenchmark.a/.so). If you followed the build steps above, this library will +(libbenchmark.a/.so). If you followed the build steps above, this library will be under the build directory you created. ```bash @@ -299,8 +299,6 @@ too (`-lkstat`). [Setting the Time Unit](#setting-the-time-unit) -[Random Interleaving](docs/random_interleaving.md) - [User-Requested Performance Counters](docs/perf_counters.md) [Preventing Optimization](#preventing-optimization) @@ -401,8 +399,8 @@ Write benchmark results to a file with the `--benchmark_out=` option (or set `BENCHMARK_OUT`). Specify the output format with `--benchmark_out_format={json|console|csv}` (or set `BENCHMARK_OUT_FORMAT={json|console|csv}`). Note that the 'csv' reporter is -deprecated and the saved `.csv` file -[is not parsable](https://github.com/google/benchmark/issues/794) by csv +deprecated and the saved `.csv` file +[is not parsable](https://github.com/google/benchmark/issues/794) by csv parsers. Specifying `--benchmark_out` does not suppress the console output. diff --git a/docs/random_interleaving.md b/docs/random_interleaving.md deleted file mode 100644 index 2471b46bb0..0000000000 --- a/docs/random_interleaving.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# Random Interleaving - -[Random Interleaving](https://github.com/google/benchmark/issues/1051) is a -technique to lower run-to-run variance. It breaks the execution of a -microbenchmark into multiple chunks and randomly interleaves them with chunks -from other microbenchmarks in the same benchmark test. Data shows it is able to -lower run-to-run variance by -[40%](https://github.com/google/benchmark/issues/1051) on average. - -To use, set `--benchmark_enable_random_interleaving=true`. - -It's a known issue that random interleaving may increase the benchmark execution -time, if: - -1. A benchmark has costly setup and / or teardown. Random interleaving will run - setup and teardown many times and may increase test execution time - significantly. -2. The time to run a single benchmark iteration is larger than the desired time - per repetition (i.e., `benchmark_min_time / benchmark_repetitions`). - -The overhead of random interleaving can be controlled by -`--benchmark_random_interleaving_max_overhead`. The default value is 0.4 meaning -the total execution time under random interlaving is limited by 1.4 x original -total execution time. Set it to `inf` for unlimited overhead. diff --git a/src/benchmark.cc b/src/benchmark.cc index 97d657a08f..d205232b8a 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -33,10 +33,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -56,18 +54,6 @@ #include "thread_manager.h" #include "thread_timer.h" -// Each benchmark can be repeated a number of times, and within each -// *repetition*, we run the user-defined benchmark function a number of -// *iterations*. The number of repetitions is determined based on flags -// (--benchmark_repetitions). -namespace { - -// Attempt to make each repetition run for at least this much of time. -constexpr double kDefaultMinTimeTotalSecs = 0.5; -constexpr int kRandomInterleavingDefaultRepetitions = 12; - -} // namespace - // Print a list of benchmarks. This option overrides all other options. DEFINE_bool(benchmark_list_tests, false); @@ -76,39 +62,16 @@ DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. DEFINE_string(benchmark_filter, "."); -// Do NOT read these flags directly. Use Get*() to read them. -namespace do_not_read_flag_directly { - -// Minimum number of seconds we should run benchmark per repetition before -// results are considered significant. For cpu-time based tests, this is the -// lower bound on the total cpu time used by all threads that make up the test. -// For real-time based tests, this is the lower bound on the elapsed time of the -// benchmark execution, regardless of number of threads. If left unset, will use -// kDefaultMinTimeTotalSecs / FLAGS_benchmark_repetitions, if random -// interleaving is enabled. Otherwise, will use kDefaultMinTimeTotalSecs. -// Do NOT read this flag directly. Use GetMinTime() to read this flag. -DEFINE_double(benchmark_min_time, -1.0); +// Minimum number of seconds we should run benchmark before results are +// considered significant. For cpu-time based tests, this is the lower bound +// on the total cpu time used by all threads that make up the test. For +// real-time based tests, this is the lower bound on the elapsed time of the +// benchmark execution, regardless of number of threads. +DEFINE_double(benchmark_min_time, 0.5); // The number of runs of each benchmark. If greater than 1, the mean and -// standard deviation of the runs will be reported. By default, the number of -// repetitions is 1 if random interleaving is disabled, and up to -// kDefaultRepetitions if random interleaving is enabled. (Read the -// documentation for random interleaving to see why it might be less than -// kDefaultRepetitions.) -// Do NOT read this flag directly, Use GetRepetitions() to access this flag. -DEFINE_int32(benchmark_repetitions, -1); - -} // namespace do_not_read_flag_directly - -// The maximum overhead allowed for random interleaving. A value X means total -// execution time under random interleaving is limited by -// (1 + X) * original total execution time. Set to 'inf' to allow infinite -// overhead. -DEFINE_double(benchmark_random_interleaving_max_overhead, 0.4); - -// If set, enable random interleaving. See -// http://github.com/google/benchmark/issues/1051 for details. -DEFINE_bool(benchmark_enable_random_interleaving, false); +// standard deviation of the runs will be reported. +DEFINE_int32(benchmark_repetitions, 1); // Report the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are reported for @@ -159,30 +122,6 @@ DEFINE_kvpairs(benchmark_context, {}); std::map* global_context = nullptr; -// Performance measurements always come with random variances. Defines a -// factor by which the required number of iterations is overestimated in order -// to reduce the probability that the minimum time requirement will not be met. -const double kSafetyMultiplier = 1.4; - -// Wraps --benchmark_min_time and returns valid default values if not supplied. -double GetMinTime() { - const double default_min_time = kDefaultMinTimeTotalSecs / GetRepetitions(); - const double flag_min_time = - do_not_read_flag_directly::FLAGS_benchmark_min_time; - return flag_min_time >= 0.0 ? flag_min_time : default_min_time; -} - -// Wraps --benchmark_repetitions and return valid default value if not supplied. -int GetRepetitions() { - const int default_repetitions = - FLAGS_benchmark_enable_random_interleaving - ? kRandomInterleavingDefaultRepetitions - : 1; - const int flag_repetitions = - do_not_read_flag_directly::FLAGS_benchmark_repetitions; - return flag_repetitions >= 0 ? flag_repetitions : default_repetitions; -} - // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} @@ -302,39 +241,6 @@ void State::FinishKeepRunning() { namespace internal { namespace { -// Flushes streams after invoking reporter methods that write to them. This -// ensures users get timely updates even when streams are not line-buffered. -void FlushStreams(BenchmarkReporter* reporter) { - if (!reporter) return; - std::flush(reporter->GetOutputStream()); - std::flush(reporter->GetErrorStream()); -} - -// Reports in both display and file reporters. -void Report(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, const RunResults& run_results) { - auto report_one = [](BenchmarkReporter* reporter, - bool aggregates_only, - const RunResults& results) { - assert(reporter); - // If there are no aggregates, do output non-aggregates. - aggregates_only &= !results.aggregates_only.empty(); - if (!aggregates_only) - reporter->ReportRuns(results.non_aggregates); - if (!results.aggregates_only.empty()) - reporter->ReportRuns(results.aggregates_only); - }; - - report_one(display_reporter, run_results.display_report_aggregates_only, - run_results); - if (file_reporter) - report_one(file_reporter, run_results.file_report_aggregates_only, - run_results); - - FlushStreams(display_reporter); - FlushStreams(file_reporter); -} - void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { @@ -342,7 +248,7 @@ void RunBenchmarks(const std::vector& benchmarks, CHECK(display_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. - bool might_have_aggregates = GetRepetitions() > 1; + bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; size_t name_field_width = 10; size_t stat_field_width = 0; for (const BenchmarkInstance& benchmark : benchmarks) { @@ -350,9 +256,8 @@ void RunBenchmarks(const std::vector& benchmarks, std::max(name_field_width, benchmark.name().str().size()); might_have_aggregates |= benchmark.repetitions() > 1; - for (const auto& Stat : benchmark.statistics()) { + for (const auto& Stat : benchmark.statistics()) stat_field_width = std::max(stat_field_width, Stat.name_.size()); - } } if (might_have_aggregates) name_field_width += 1 + stat_field_width; @@ -363,61 +268,45 @@ void RunBenchmarks(const std::vector& benchmarks, // Keep track of running times of all instances of current benchmark std::vector complexity_reports; + // We flush streams after invoking reporter methods that write to them. This + // ensures users get timely updates even when streams are not line-buffered. + auto flushStreams = [](BenchmarkReporter* reporter) { + if (!reporter) return; + std::flush(reporter->GetOutputStream()); + std::flush(reporter->GetErrorStream()); + }; + if (display_reporter->ReportContext(context) && (!file_reporter || file_reporter->ReportContext(context))) { - FlushStreams(display_reporter); - FlushStreams(file_reporter); - - // Without random interleaving, benchmarks are executed in the order of: - // A, A, ..., A, B, B, ..., B, C, C, ..., C, ... - // That is, repetition is within RunBenchmark(), hence the name - // inner_repetitions. - // With random interleaving, benchmarks are executed in the order of: - // {Random order of A, B, C, ...}, {Random order of A, B, C, ...}, ... - // That is, repetitions is outside of RunBenchmark(), hence the name - // outer_repetitions. - int inner_repetitions = - FLAGS_benchmark_enable_random_interleaving ? 1 : GetRepetitions(); - int outer_repetitions = - FLAGS_benchmark_enable_random_interleaving ? GetRepetitions() : 1; - std::vector benchmark_indices(benchmarks.size()); - for (size_t i = 0; i < benchmarks.size(); ++i) { - benchmark_indices[i] = i; - } - - std::random_device rd; - std::mt19937 g(rd()); - // 'run_results_vector' and 'benchmarks' are parallel arrays. - std::vector run_results_vector(benchmarks.size()); - for (int i = 0; i < outer_repetitions; i++) { - if (FLAGS_benchmark_enable_random_interleaving) { - std::shuffle(benchmark_indices.begin(), benchmark_indices.end(), g); - } - for (size_t j : benchmark_indices) { - // Repetitions will be automatically adjusted under random interleaving. - if (!FLAGS_benchmark_enable_random_interleaving || - i < benchmarks[j].RandomInterleavingRepetitions()) { - RunBenchmark(benchmarks[j], outer_repetitions, inner_repetitions, - &complexity_reports, &run_results_vector[j]); - if (!FLAGS_benchmark_enable_random_interleaving) { - // Print out reports as they come in. - Report(display_reporter, file_reporter, run_results_vector.at(j)); - } - } - } - } - - if (FLAGS_benchmark_enable_random_interleaving) { - // Print out all reports at the end of the test. - for (const RunResults& run_results : run_results_vector) { - Report(display_reporter, file_reporter, run_results); - } + flushStreams(display_reporter); + flushStreams(file_reporter); + + for (const auto& benchmark : benchmarks) { + RunResults run_results = RunBenchmark(benchmark, &complexity_reports); + + auto report = [&run_results](BenchmarkReporter* reporter, + bool report_aggregates_only) { + assert(reporter); + // If there are no aggregates, do output non-aggregates. + report_aggregates_only &= !run_results.aggregates_only.empty(); + if (!report_aggregates_only) + reporter->ReportRuns(run_results.non_aggregates); + if (!run_results.aggregates_only.empty()) + reporter->ReportRuns(run_results.aggregates_only); + }; + + report(display_reporter, run_results.display_report_aggregates_only); + if (file_reporter) + report(file_reporter, run_results.file_report_aggregates_only); + + flushStreams(display_reporter); + flushStreams(file_reporter); } } display_reporter->Finalize(); if (file_reporter) file_reporter->Finalize(); - FlushStreams(display_reporter); - FlushStreams(file_reporter); + flushStreams(display_reporter); + flushStreams(file_reporter); } // Disable deprecated warnings temporarily because we need to reference @@ -567,7 +456,6 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" - " [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n" " [--benchmark_format=]\n" @@ -588,16 +476,10 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag( - argv[i], "benchmark_min_time", - &do_not_read_flag_directly::FLAGS_benchmark_min_time) || - ParseInt32Flag( - argv[i], "benchmark_repetitions", - &do_not_read_flag_directly::FLAGS_benchmark_repetitions) || - ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", - &FLAGS_benchmark_enable_random_interleaving) || - ParseDoubleFlag(argv[i], "benchmark_random_interleaving_max_overhead", - &FLAGS_benchmark_random_interleaving_max_overhead) || + ParseDoubleFlag(argv[i], "benchmark_min_time", + &FLAGS_benchmark_min_time) || + ParseInt32Flag(argv[i], "benchmark_repetitions", + &FLAGS_benchmark_repetitions) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", &FLAGS_benchmark_report_aggregates_only) || ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", diff --git a/src/benchmark_adjust_repetitions.cc b/src/benchmark_adjust_repetitions.cc deleted file mode 100644 index 2847927628..0000000000 --- a/src/benchmark_adjust_repetitions.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "benchmark_adjust_repetitions.h" - -#include "benchmark_api_internal.h" -#include "log.h" - -namespace benchmark { -namespace internal { - -namespace { - -constexpr double kNanosecondInSecond = 1e-9; - -} // namespace - -int ComputeRandomInterleavingRepetitions( - InternalRandomInterleavingRepetitionsInput input) { - // Find the repetitions such that total overhead is bounded. Let - // n = desired number of repetitions, i.e., the output of this method. - // t = total real execution time per repetition including overhead, - // (input.total_execution_time_per_repetition). - // o = maximum allowed increase in total real execution time due to random - // interleaving, measured as a fraction (input.max_overhead). - // e = estimated total execution time without Random Interleaving - // We want - // t * n / e <= 1 + o - // I.e., - // n <= (1 + o) * e / t - // - // Let - // h = overhead per repetition, which include all setup / teardown time and - // also the execution time of preliminary trials used to search for the - // correct number of iterations. - // r = real execution time per repetition not including overhead - // (input.real_accumulated_time_per_repetition). - // s = measured execution time per repetition not including overhead, - // which can be either real or CPU time - // (input.accumulated_time_per_repetition). - // We have - // h = t - r - // - // Let - // m = total minimum measured execution time for all repetitions - // (input.min_time_per_repetition * input.max_repetitions). - // Let - // f = m / s - // f is the scale factor between m and s, and will be used to estimate - // l, the total real execution time for all repetitions excluding the - // overhead. It's reasonable to assume that the real execution time excluding - // the overhead is proportional to the measured time. Hence we expect to see - // l / r to be equal to m / s. That is, l / r = f, thus, l = r * f. Then the - // total execution time e can be estimated by h + l, which is h + r * f. - // e = h + r * f - // Note that this might be an underestimation. If number of repetitions is - // reduced, we may need to run more iterations per repetition, and that may - // increase the number of preliminary trials needed to find the correct - // number of iterations. - - double h = std::max(0.0, input.total_execution_time_per_repetition - - input.real_time_used_per_repetition); - double r = - std::max(input.real_time_used_per_repetition, kNanosecondInSecond); - double s = - std::max(input.time_used_per_repetition, kNanosecondInSecond); - double m = input.min_time_per_repetition * input.max_repetitions; - - // f = m / s - // RunBenchmark() always overshoot the iteration count by kSafetyMultiplier. - // Apply the same factor here. - // f = kSafetyMultiplier * m / s - // Also we want to make sure 1 <= f <= input.max_repetitions. Note that we - // may not be able to reach m because the total iters per repetition is - // upper bounded by --benchmark_max_iters. This behavior is preserved in - // Random Interleaving, as we won't run repetitions more than - // input.max_repetitions to reach m. - - double f = kSafetyMultiplier * m / s; - f = std::min(std::max(f, 1.0), static_cast(input.max_repetitions)); - - double e = h + r * f; - // n <= (1 + o) * e / t = (1 + o) * e / (h + r) - // Also we want to make sure 1 <= n <= input.max_repetition, and (h + r) > 0. - double n = (1 + input.max_overhead) * e / (h + r); - n = std::min(std::max(n, 1.0), static_cast(input.max_repetitions)); - - int n_int = static_cast(n); - - VLOG(2) << "Computed random interleaving repetitions" - << "\n input.total_execution_time_per_repetition: " - << input.total_execution_time_per_repetition - << "\n input.time_used_per_repetition: " - << input.time_used_per_repetition - << "\n input.real_time_used_per_repetition: " - << input.real_time_used_per_repetition - << "\n input.min_time_per_repetitions: " - << input.min_time_per_repetition - << "\n input.max_repetitions: " << input.max_repetitions - << "\n input.max_overhead: " << input.max_overhead - << "\n h: " << h - << "\n r: " << r - << "\n s: " << s - << "\n f: " << f - << "\n m: " << m - << "\n e: " << e - << "\n n: " << n - << "\n n_int: " << n_int; - - return n_int; -} - -} // internal -} // benchmark diff --git a/src/benchmark_adjust_repetitions.h b/src/benchmark_adjust_repetitions.h deleted file mode 100644 index 21a666afe0..0000000000 --- a/src/benchmark_adjust_repetitions.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef BENCHMARK_ADJUST_REPETITIONS_H -#define BENCHMARK_ADJUST_REPETITIONS_H - -#include "benchmark/benchmark.h" -#include "commandlineflags.h" - -namespace benchmark { -namespace internal { - -// Defines the input tuple to ComputeRandomInterleavingRepetitions(). -struct InternalRandomInterleavingRepetitionsInput { - double total_execution_time_per_repetition; - double time_used_per_repetition; - double real_time_used_per_repetition; - double min_time_per_repetition; - double max_overhead; - int max_repetitions; -}; - -// Should be called right after the first repetition is completed to estimate -// the number of iterations. -int ComputeRandomInterleavingRepetitions( - InternalRandomInterleavingRepetitionsInput input); - -} // end namespace internal -} // end namespace benchmark - -#endif // BENCHMARK_ADJUST_REPETITIONS_H diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index ddd46bee63..553ff44e5c 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -2,11 +2,8 @@ #include -#include "check.h" #include "string_util.h" -DECLARE_bool(benchmark_enable_random_interleaving); - namespace benchmark { namespace internal { @@ -24,12 +21,9 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, complexity_lambda_(benchmark_.complexity_lambda_), statistics_(benchmark_.statistics_), repetitions_(benchmark_.repetitions_), - min_time_(!IsZero(benchmark_.min_time_) ? benchmark_.min_time_ - : GetMinTime()), + min_time_(benchmark_.min_time_), iterations_(benchmark_.iterations_), threads_(thread_count) { - CHECK(!IsZero(min_time_)) << "min_time must be non-zero."; - name_.function_name = benchmark_.name_; size_t arg_i = 0; @@ -83,32 +77,6 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, } } -double BenchmarkInstance::MinTime() const { - if (FLAGS_benchmark_enable_random_interleaving) { - // Random Interleaving will automatically adjust - // random_interleaving_repetitions(). Dividing - // total execution time by random_interleaving_repetitions() gives - // the adjusted min_time per repetition. - return min_time_ * GetRepetitions() / RandomInterleavingRepetitions(); - } - return min_time_; -} - -int BenchmarkInstance::RandomInterleavingRepetitions() const { - return random_interleaving_repetitions_ < 0 - ? GetRepetitions() - : random_interleaving_repetitions_; -} - -bool BenchmarkInstance::RandomInterleavingRepetitionsInitialized() const { - return random_interleaving_repetitions_ >= 0; -} - -void BenchmarkInstance::InitRandomInterleavingRepetitions( - int reps) const { - random_interleaving_repetitions_ = reps; -} - State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 0ff8dafbe6..b0595e5a12 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -1,10 +1,6 @@ #ifndef BENCHMARK_API_INTERNAL_H #define BENCHMARK_API_INTERNAL_H -#include "benchmark/benchmark.h" -#include "commandlineflags.h" - -#include #include #include #include @@ -18,25 +14,12 @@ namespace benchmark { namespace internal { -extern const double kSafetyMultiplier; - // Information kept per benchmark we may want to run class BenchmarkInstance { public: BenchmarkInstance(Benchmark* benchmark, const std::vector& args, int threads); - // Returns number of repetitions for Random Interleaving. This will be - // initialized later once we finish the first repetition, if Random - // Interleaving is enabled. See also ComputeRandominterleavingrepetitions(). - int RandomInterleavingRepetitions() const; - - // Returns true if repetitions for Random Interleaving is initialized. - bool RandomInterleavingRepetitionsInitialized() const; - - // Initializes number of repetitions for random interleaving. - void InitRandomInterleavingRepetitions(int reps) const; - const BenchmarkName& name() const { return name_; } AggregationReportMode aggregation_report_mode() const { return aggregation_report_mode_; @@ -49,7 +32,7 @@ class BenchmarkInstance { BigOFunc& complexity_lambda() const { return *complexity_lambda_; } const std::vector& statistics() const { return statistics_; } int repetitions() const { return repetitions_; } - double MinTime() const; + double min_time() const { return min_time_; } IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } @@ -70,13 +53,12 @@ class BenchmarkInstance { bool use_manual_time_; BigO complexity_; BigOFunc* complexity_lambda_; - std::vector statistics_; + UserCounters counters_; + const std::vector& statistics_; int repetitions_; double min_time_; IterationCount iterations_; - int threads_; - UserCounters counters_; - mutable int random_interleaving_repetitions_ = -1; + int threads_; // Number of concurrent threads to us }; bool FindBenchmarksInternal(const std::string& re, @@ -87,10 +69,6 @@ bool IsZero(double n); ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); -double GetMinTime(); - -int GetRepetitions(); - } // end namespace internal } // end namespace benchmark diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 330cb4494b..3cffbbfad5 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -15,7 +15,6 @@ #include "benchmark_runner.h" #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" -#include "benchmark_adjust_repetitions.h" #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS @@ -53,9 +52,6 @@ #include "thread_manager.h" #include "thread_timer.h" -DECLARE_bool(benchmark_enable_random_interleaving); -DECLARE_double(benchmark_random_interleaving_max_overhead); - namespace benchmark { namespace internal { @@ -71,7 +67,7 @@ BenchmarkReporter::Run CreateRunReport( const internal::ThreadManager::Result& results, IterationCount memory_iterations, const MemoryManager::Result& memory_result, double seconds, - int repetition_index) { + int64_t repetition_index) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -142,16 +138,12 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, - int outer_repetitions_, - int inner_repetitions_, - std::vector* complexity_reports_, - RunResults* run_results_) + std::vector* complexity_reports_) : b(b_), complexity_reports(*complexity_reports_), - run_results(*run_results_), - outer_repetitions(outer_repetitions_), - inner_repetitions(inner_repetitions_), - repeats(b.repetitions() != 0 ? b.repetitions() : inner_repetitions), + min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + repeats(b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations() != 0), pool(b.threads() - 1), iters(has_explicit_iteration_count ? b.iterations() : 1), @@ -171,7 +163,6 @@ class BenchmarkRunner { internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); - CHECK(FLAGS_benchmark_perf_counters.empty() || perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; @@ -188,20 +179,21 @@ class BenchmarkRunner { if ((b.complexity() != oNone) && b.last_benchmark_instance) { auto additional_run_stats = ComputeBigO(complexity_reports); run_results.aggregates_only.insert(run_results.aggregates_only.end(), - additional_run_stats.begin(), - additional_run_stats.end()); + additional_run_stats.begin(), + additional_run_stats.end()); complexity_reports.clear(); } } + RunResults&& get_results() { return std::move(run_results); } + private: + RunResults run_results; + const benchmark::internal::BenchmarkInstance& b; std::vector& complexity_reports; - RunResults& run_results; - - const int outer_repetitions; - const int inner_repetitions; + const double min_time; const int repeats; const bool has_explicit_iteration_count; @@ -276,14 +268,13 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = - b.MinTime() * kSafetyMultiplier / std::max(i.seconds, 1e-9); + double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); // If our last run was at least 10% of FLAGS_benchmark_min_time then we // use the multiplier directly. // Otherwise we use at most 10 times expansion. // NOTE: When the last run was at least 10% of the min time the max // expansion should be 14x. - bool is_significant = (i.seconds / b.MinTime()) > 0.1; + bool is_significant = (i.seconds / min_time) > 0.1; multiplier = is_significant ? multiplier : std::min(10.0, multiplier); if (multiplier <= 1.0) multiplier = 2.0; @@ -304,17 +295,14 @@ class BenchmarkRunner { // or because an error was reported. return i.results.has_error_ || i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= b.MinTime() || // The elapsed time is large enough. - // CPU time is specified but the - // elapsed real time greatly exceeds - // the minimum time. Note that user - // provided timers are except from this - // sanity check. - ((i.results.real_time_used >= 5 * b.MinTime()) && - !b.use_manual_time()); + i.seconds >= min_time || // The elapsed time is large enough. + // CPU time is specified but the elapsed real time greatly exceeds + // the minimum time. + // Note that user provided timers are except from this sanity check. + ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); } - void DoOneRepetition(int repetition_index) { + void DoOneRepetition(int64_t repetition_index) { const bool is_the_first_repetition = repetition_index == 0; IterationResults i; @@ -324,10 +312,8 @@ class BenchmarkRunner { // Please do note that the if there are repetitions, the iteration count // is *only* calculated for the *first* repetition, and other repetitions // simply use that precomputed iteration count. - const auto exec_start = benchmark::ChronoClockNow(); for (;;) { i = DoNIterations(); - const auto exec_end = benchmark::ChronoClockNow(); // Do we consider the results to be significant? // If we are doing repetitions, and the first repetition was already done, @@ -338,38 +324,7 @@ class BenchmarkRunner { has_explicit_iteration_count || ShouldReportIterationResults(i); - if (results_are_significant) { - // The number of repetitions for random interleaving may be reduced - // to limit the increase in benchmark execution time. When this happens - // the target execution time for each repetition is increased. We may - // need to rerun trials to calculate iters according to the increased - // target execution time. - bool rerun_trial = false; - // If random interleaving is enabled and the repetitions is not - // initialized, do it now. - if (FLAGS_benchmark_enable_random_interleaving && - !b.RandomInterleavingRepetitionsInitialized()) { - InternalRandomInterleavingRepetitionsInput input; - input.total_execution_time_per_repetition = exec_end - exec_start; - input.time_used_per_repetition = i.seconds; - input.real_time_used_per_repetition = i.results.real_time_used; - input.min_time_per_repetition = GetMinTime(); - input.max_overhead = FLAGS_benchmark_random_interleaving_max_overhead; - input.max_repetitions = GetRepetitions(); - b.InitRandomInterleavingRepetitions( - ComputeRandomInterleavingRepetitions(input)); - // If the number of repetitions changed, need to rerun the last trial - // because iters may also change. Note that we only need to do this - // if accumulated_time < b.MinTime(), i.e., the iterations we have - // run is not enough for the already adjusted b.MinTime(). - // Otherwise, we will still skip the rerun. - rerun_trial = - b.RandomInterleavingRepetitions() < GetRepetitions() && - i.seconds < b.MinTime() && !has_explicit_iteration_count; - } - - if (!rerun_trial) break; // Good, let's report them! - } + if (results_are_significant) break; // Good, let's report them! // Nope, bad iteration. Let's re-estimate the hopefully-sufficient // iteration count, and run the benchmark again... @@ -386,8 +341,7 @@ class BenchmarkRunner { if (memory_manager != nullptr) { // Only run a few iterations to reduce the impact of one-time // allocations in benchmarks that are not properly managed. - memory_iterations = std::min( - 16 / outer_repetitions + (16 % outer_repetitions != 0), iters); + memory_iterations = std::min(16, iters); memory_manager->Start(); std::unique_ptr manager; manager.reset(new internal::ThreadManager(1)); @@ -413,12 +367,11 @@ class BenchmarkRunner { } // end namespace -void RunBenchmark(const benchmark::internal::BenchmarkInstance& b, - const int outer_repetitions, const int inner_repetitions, - std::vector* complexity_reports, - RunResults* run_results) { - internal::BenchmarkRunner r(b, outer_repetitions, inner_repetitions, - complexity_reports, run_results); +RunResults RunBenchmark( + const benchmark::internal::BenchmarkInstance& b, + std::vector* complexity_reports) { + internal::BenchmarkRunner r(b, complexity_reports); + return r.get_results(); } } // end namespace internal diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index e29aa32306..9b0cf2a64e 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -42,10 +42,9 @@ struct RunResults { bool file_report_aggregates_only = false; }; -void RunBenchmark(const benchmark::internal::BenchmarkInstance& b, - int outer_repetitions, int inner_repetitions, - std::vector* complexity_reports, - RunResults* run_results); +RunResults RunBenchmark( + const benchmark::internal::BenchmarkInstance& b, + std::vector* complexity_reports); } // namespace internal diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8945f5005f..1e7b6829f2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -196,7 +196,6 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(benchmark_gtest) add_gtest(benchmark_name_gtest) - add_gtest(benchmark_random_interleaving_gtest) add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc deleted file mode 100644 index 5e8329a4e6..0000000000 --- a/test/benchmark_random_interleaving_gtest.cc +++ /dev/null @@ -1,271 +0,0 @@ -#include -#include -#include - -#include "../src/benchmark_adjust_repetitions.h" -#include "../src/string_util.h" -#include "benchmark/benchmark.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -DECLARE_bool(benchmark_enable_random_interleaving); -DECLARE_string(benchmark_filter); -DECLARE_double(benchmark_random_interleaving_max_overhead); - -namespace do_not_read_flag_directly { -DECLARE_int32(benchmark_repetitions); -} // namespace do_not_read_flag_directly - -namespace benchmark { -namespace internal { -namespace { - -class EventQueue : public std::queue { - public: - void Put(const std::string& event) { - push(event); - } - - void Clear() { - while (!empty()) { - pop(); - } - } - - std::string Get() { - std::string event = front(); - pop(); - return event; - } -}; - -static EventQueue* queue = new EventQueue; - -class NullReporter : public BenchmarkReporter { - public: - bool ReportContext(const Context& /*context*/) override { - return true; - } - void ReportRuns(const std::vector& /* report */) override {} -}; - -class BenchmarkTest : public testing::Test { - public: - static void SetupHook(int /* num_threads */) { queue->push("Setup"); } - - static void TeardownHook(int /* num_threads */) { queue->push("Teardown"); } - - void Execute(const std::string& pattern) { - queue->Clear(); - - BenchmarkReporter* reporter = new NullReporter; - FLAGS_benchmark_filter = pattern; - RunSpecifiedBenchmarks(reporter); - delete reporter; - - queue->Put("DONE"); // End marker - } -}; - -static void BM_Match1(benchmark::State& state) { - const int64_t arg = state.range(0); - - for (auto _ : state) {} - queue->Put(StrFormat("BM_Match1/%d", static_cast(arg))); -} -BENCHMARK(BM_Match1) - ->Iterations(100) - ->Arg(1) - ->Arg(2) - ->Arg(3) - ->Range(10, 80) - ->Args({90}) - ->Args({100}); - -static void BM_MatchOverhead(benchmark::State& state) { - const int64_t arg = state.range(0); - - for (auto _ : state) {} - queue->Put(StrFormat("BM_MatchOverhead/%d", static_cast(arg))); -} -BENCHMARK(BM_MatchOverhead) - ->Iterations(100) - ->Arg(64) - ->Arg(80); - -TEST_F(BenchmarkTest, Match1) { - Execute("BM_Match1"); - ASSERT_EQ("BM_Match1/1", queue->Get()); - ASSERT_EQ("BM_Match1/2", queue->Get()); - ASSERT_EQ("BM_Match1/3", queue->Get()); - ASSERT_EQ("BM_Match1/10", queue->Get()); - ASSERT_EQ("BM_Match1/64", queue->Get()); - ASSERT_EQ("BM_Match1/80", queue->Get()); - ASSERT_EQ("BM_Match1/90", queue->Get()); - ASSERT_EQ("BM_Match1/100", queue->Get()); - ASSERT_EQ("DONE", queue->Get()); -} - -TEST_F(BenchmarkTest, Match1WithRepetition) { - do_not_read_flag_directly::FLAGS_benchmark_repetitions = 2; - - Execute("BM_Match1/(64|80)"); - ASSERT_EQ("BM_Match1/64", queue->Get()); - ASSERT_EQ("BM_Match1/64", queue->Get()); - ASSERT_EQ("BM_Match1/80", queue->Get()); - ASSERT_EQ("BM_Match1/80", queue->Get()); - ASSERT_EQ("DONE", queue->Get()); -} - -TEST_F(BenchmarkTest, Match1WithRandomInterleaving) { - FLAGS_benchmark_enable_random_interleaving = true; - do_not_read_flag_directly::FLAGS_benchmark_repetitions = 100; - FLAGS_benchmark_random_interleaving_max_overhead = - std::numeric_limits::infinity(); - - std::vector expected({"BM_Match1/64", "BM_Match1/80"}); - std::map interleaving_count; - Execute("BM_Match1/(64|80)"); - for (int i = 0; i < 100; ++i) { - std::vector interleaving; - interleaving.push_back(queue->Get()); - interleaving.push_back(queue->Get()); - EXPECT_THAT(interleaving, testing::UnorderedElementsAreArray(expected)); - interleaving_count[StrFormat("%s,%s", interleaving[0].c_str(), - interleaving[1].c_str())]++; - } - EXPECT_GE(interleaving_count.size(), 2) << "Interleaving was not randomized."; - ASSERT_EQ("DONE", queue->Get()); -} - -TEST_F(BenchmarkTest, Match1WithRandomInterleavingAndZeroOverhead) { - FLAGS_benchmark_enable_random_interleaving = true; - do_not_read_flag_directly::FLAGS_benchmark_repetitions = 100; - FLAGS_benchmark_random_interleaving_max_overhead = 0; - - // ComputeRandomInterleavingRepetitions() will kick in and rerun each - // benchmark once with increased iterations. Then number of repetitions will - // be reduced to < 100. The first 4 executions should be - // 2 x BM_MatchOverhead/64 and 2 x BM_MatchOverhead/80. - std::vector expected( - {"BM_MatchOverhead/64", "BM_MatchOverhead/80", "BM_MatchOverhead/64", - "BM_MatchOverhead/80"}); - std::map interleaving_count; - Execute("BM_MatchOverhead/(64|80)"); - std::vector interleaving; - interleaving.push_back(queue->Get()); - interleaving.push_back(queue->Get()); - interleaving.push_back(queue->Get()); - interleaving.push_back(queue->Get()); - EXPECT_THAT(interleaving, testing::UnorderedElementsAreArray(expected)); - ASSERT_LT(queue->size(), 100) << "# Repetitions was not reduced to < 100."; -} - -InternalRandomInterleavingRepetitionsInput CreateInput( - double total, double time, double real_time, double min_time, - double overhead, int repetitions) { - InternalRandomInterleavingRepetitionsInput input; - input.total_execution_time_per_repetition = total; - input.time_used_per_repetition = time; - input.real_time_used_per_repetition = real_time; - input.min_time_per_repetition = min_time; - input.max_overhead = overhead; - input.max_repetitions = repetitions; - return input; -} - -TEST(Benchmark, ComputeRandomInterleavingRepetitions) { - // On wall clock time. - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.05, 0.05, 0.05, 0.05, 0.0, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.05, 0.05, 0.05, 0.05, 0.4, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.06, 0.05, 0.05, 0.05, 0.0, 10)), - 8); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.06, 0.05, 0.05, 0.05, 0.4, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.08, 0.05, 0.05, 0.05, 0.0, 10)), - 6); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.08, 0.05, 0.05, 0.05, 0.4, 10)), - 9); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.26, 0.25, 0.25, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.25, 0.25, 0.25, 0.05, 0.4, 10)), - 3); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.26, 0.25, 0.25, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.26, 0.25, 0.25, 0.05, 0.4, 10)), - 3); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.38, 0.25, 0.25, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.38, 0.25, 0.25, 0.05, 0.4, 10)), - 3); - - // On CPU time. - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.1, 0.05, 0.1, 0.05, 0.0, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.1, 0.05, 0.1, 0.05, 0.4, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.11, 0.05, 0.1, 0.05, 0.0, 10)), - 9); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.11, 0.05, 0.1, 0.05, 0.4, 10)), - 10); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.15, 0.05, 0.1, 0.05, 0.0, 10)), - 7); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.15, 0.05, 0.1, 0.05, 0.4, 10)), - 9); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.5, 0.25, 0.5, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.5, 0.25, 0.5, 0.05, 0.4, 10)), - 3); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.51, 0.25, 0.5, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.51, 0.25, 0.5, 0.05, 0.4, 10)), - 3); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.8, 0.25, 0.5, 0.05, 0.0, 10)), - 2); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.8, 0.25, 0.5, 0.05, 0.4, 10)), - 2); - - // Corner cases. - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.0, 0.25, 0.5, 0.05, 0.4, 10)), - 3); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.8, 0.0, 0.5, 0.05, 0.4, 10)), - 9); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.8, 0.25, 0.0, 0.05, 0.4, 10)), - 1); - EXPECT_EQ(ComputeRandomInterleavingRepetitions( - CreateInput(0.8, 0.25, 0.5, 0.0, 0.4, 10)), - 1); -} - -} // namespace -} // namespace internal -} // namespace benchmark From a54ef37aea23d36ec8f38500581578817c0f9c9b Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 2 Jun 2021 12:34:00 +0300 Subject: [PATCH 311/330] Ensure that we print repetition count even when it was specified via flag `--benchmark_repetitions=` --- src/benchmark_runner.cc | 6 +- test/CMakeLists.txt | 3 + test/memory_manager_test.cc | 2 +- test/repetitions_test.cc | 186 +++++++++++++++++++++++++++++ test/reporter_output_test.cc | 28 ++--- test/user_counters_tabular_test.cc | 10 +- test/user_counters_test.cc | 24 ++-- 7 files changed, 224 insertions(+), 35 deletions(-) create mode 100644 test/repetitions_test.cc diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 3cffbbfad5..d4eb7d9f4e 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -67,7 +67,7 @@ BenchmarkReporter::Run CreateRunReport( const internal::ThreadManager::Result& results, IterationCount memory_iterations, const MemoryManager::Result& memory_result, double seconds, - int64_t repetition_index) { + int64_t repetition_index, int64_t repeats) { // Create report about this benchmark run. BenchmarkReporter::Run report; @@ -80,7 +80,7 @@ BenchmarkReporter::Run CreateRunReport( report.time_unit = b.time_unit(); report.threads = b.threads(); report.repetition_index = repetition_index; - report.repetitions = b.repetitions(); + report.repetitions = repeats; if (!report.error_occurred) { if (b.use_manual_time()) { @@ -356,7 +356,7 @@ class BenchmarkRunner { // Ok, now actualy report. BenchmarkReporter::Run report = CreateRunReport(b, i.results, memory_iterations, memory_result, - i.seconds, repetition_index); + i.seconds, repetition_index, repeats); if (!report.error_occurred && b.complexity() != oNone) complexity_reports.push_back(report); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1e7b6829f2..012f5a8bcd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -87,6 +87,9 @@ add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +compile_output_test(repetitions_test) +add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) + compile_benchmark_test(diagnostics_test) add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 71d4d0ec9c..4e0e6495ba 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -23,7 +23,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc new file mode 100644 index 0000000000..5980c99367 --- /dev/null +++ b/test/repetitions_test.cc @@ -0,0 +1,186 @@ + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// ========================================================================= // +// ------------------------ Testing Basic Output --------------------------- // +// ========================================================================= // + +void BM_ExplicitRepetitions(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_ExplicitRepetitions)->Repetitions(2); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_ExplicitRepetitions/repeats:2 %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ExplicitRepetitions/repeats:2 %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ExplicitRepetitions/repeats:2_mean %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ExplicitRepetitions/repeats:2_median %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ExplicitRepetitions/repeats:2_stddev %console_report$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"}, + {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ExplicitRepetitions/repeats:2\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_ExplicitRepetitions/repeats:2_mean\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_ExplicitRepetitions/repeats:2_median\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_ExplicitRepetitions/repeats:2_stddev\",%csv_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Basic Output --------------------------- // +// ========================================================================= // + +void BM_ImplicitRepetitions(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_ImplicitRepetitions); + +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions %console_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_mean %console_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_median %console_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_stddev %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"repetition_index\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"}, + {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 3,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_mean\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_median\",%csv_report$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_stddev\",%csv_report$"}}); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index b3d8edbae0..31bedfa7cd 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -73,7 +73,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -101,7 +101,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -130,7 +130,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -156,7 +156,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -183,7 +183,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_nanosecond\",$"}, {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -204,7 +204,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_microsecond\",$"}, {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -225,7 +225,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_millisecond\",$"}, {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -245,7 +245,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"}, {"\"run_name\": \"BM_time_label_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -269,7 +269,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, @@ -291,7 +291,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); @@ -309,7 +309,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); @@ -329,7 +329,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); @@ -348,7 +348,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"}, {"\"run_name\": \"BM_custom_name\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -833,7 +833,7 @@ BENCHMARK(BM_JSON_Format); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 18373c0aac..d49ce55fbb 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -73,7 +73,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -126,7 +126,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -176,7 +176,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -214,7 +214,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -256,7 +256,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 5699f4f5e1..4d53f5c424 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -34,7 +34,7 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -80,7 +80,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -127,7 +127,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -167,7 +167,7 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, {"\"run_name\": \"BM_Invert\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -209,7 +209,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_InvertedRate\",$"}, {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -248,7 +248,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -287,7 +287,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -330,7 +330,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -369,7 +369,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -415,7 +415,7 @@ ADD_CASES(TC_JSONOut, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -457,7 +457,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -500,7 +500,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 0,$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, From e0a080d00ea013446fcd3579760d70236690ebe4 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 2 Jun 2021 13:28:05 +0300 Subject: [PATCH 312/330] BenchmarkFamilies::FindBenchmarks(): correctly use std::vector<>::reserve() It takes the whole total new capacity, not the increase. --- src/benchmark_register.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 1f0dcd1d0e..ebea2d92cc 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -150,7 +150,7 @@ bool BenchmarkFamilies::FindBenchmarks( } // reserve in the special case the regex ".", since we know the final // family size. - if (spec == ".") benchmarks->reserve(family_size); + if (spec == ".") benchmarks->reserve(benchmarks->size() + family_size); for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { From 4c2e32f1d02a18074c60252aab7bf97e76f32db2 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 2 Jun 2021 18:06:45 +0300 Subject: [PATCH 313/330] Introduce "family index" field into JSON output (#1164) It may be useful for those wishing to further post-process JSON results, but it is mainly geared towards better support for run interleaving, where results from the same family may not be close-by in the JSON. While we won't be able to do much about that for outputs, the tools can and perhaps should reorder the results to that at least in their output they are in proper order, not run order. Note that this only counts the families that were filtered-in, so if e.g. there were three families, and we filtered-out the second one, the two families (which were first and third) will have family indexes 0 and 1. --- include/benchmark/benchmark.h | 1 + src/benchmark_api_internal.cc | 3 +- src/benchmark_api_internal.h | 6 ++- src/benchmark_register.cc | 11 ++++- src/benchmark_runner.cc | 1 + src/complexity.cc | 2 + src/json_reporter.cc | 1 + src/statistics.cc | 1 + test/BUILD | 1 + test/complexity_test.cc | 63 +++++++++++++++------------- test/filter_test.cc | 22 ++++++++-- test/memory_manager_test.cc | 1 + test/repetitions_test.cc | 11 +++++ test/reporter_output_test.cc | 49 ++++++++++++++++++++++ test/user_counters_tabular_test.cc | 5 +++ test/user_counters_test.cc | 12 ++++++ test/user_counters_thousands_test.cc | 5 +++ 17 files changed, 159 insertions(+), 36 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 58d9237535..693c2b82f5 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1418,6 +1418,7 @@ class BenchmarkReporter { std::string benchmark_name() const; BenchmarkName run_name; + int64_t family_index; RunType run_type; std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index 553ff44e5c..c4813172c5 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -7,10 +7,11 @@ namespace benchmark { namespace internal { -BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, +BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, const std::vector& args, int thread_count) : benchmark_(*benchmark), + family_index_(family_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), time_unit_(benchmark_.time_unit_), diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index b0595e5a12..5ee6fadf04 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -17,10 +17,11 @@ namespace internal { // Information kept per benchmark we may want to run class BenchmarkInstance { public: - BenchmarkInstance(Benchmark* benchmark, const std::vector& args, - int threads); + BenchmarkInstance(Benchmark* benchmark, int family_index, + const std::vector& args, int threads); const BenchmarkName& name() const { return name_; } + int family_index() const { return family_index_; } AggregationReportMode aggregation_report_mode() const { return aggregation_report_mode_; } @@ -45,6 +46,7 @@ class BenchmarkInstance { private: BenchmarkName name_; Benchmark& benchmark_; + const int family_index_; AggregationReportMode aggregation_report_mode_; const std::vector& args_; TimeUnit time_unit_; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index ebea2d92cc..bc6c66869b 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -129,8 +129,12 @@ bool BenchmarkFamilies::FindBenchmarks( // Special list of thread counts to use when none are specified const std::vector one_thread = {1}; + int next_family_index = 0; + MutexLock l(mutex_); for (std::unique_ptr& family : families_) { + int family_index = next_family_index; + // Family was deleted or benchmark doesn't match if (!family) continue; @@ -154,13 +158,18 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { - BenchmarkInstance instance(family.get(), args, num_threads); + BenchmarkInstance instance(family.get(), family_index, args, + num_threads); const auto full_name = instance.name().str(); if ((re.Match(full_name) && !isNegativeFilter) || (!re.Match(full_name) && isNegativeFilter)) { instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); + + // Only bump the next family index once we've estabilished that + // at least one instance of this family will be run. + if (next_family_index == family_index) ++next_family_index; } } } diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index d4eb7d9f4e..c71fc719dc 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -72,6 +72,7 @@ BenchmarkReporter::Run CreateRunReport( BenchmarkReporter::Run report; report.run_name = b.name(); + report.family_index = b.family_index(); report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; diff --git a/src/complexity.cc b/src/complexity.cc index aeed67f0c7..22f4263686 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -193,6 +193,7 @@ std::vector ComputeBigO( // Get the data from the accumulator to BenchmarkReporter::Run's. Run big_o; big_o.run_name = run_name; + big_o.family_index = reports[0].family_index; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; big_o.repetitions = reports[0].repetitions; big_o.repetition_index = Run::no_repetition_index; @@ -215,6 +216,7 @@ std::vector ComputeBigO( // Only add label to mean/stddev if it is same for all runs Run rms; rms.run_name = run_name; + rms.family_index = reports[0].family_index; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index effa6bd12e..5a7d843d3c 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -207,6 +207,7 @@ void JSONReporter::PrintRunData(Run const& run) { std::string indent(6, ' '); std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; + out << indent << FormatKV("family_index", run.family_index) << ",\n"; out << indent << FormatKV("run_name", run.run_name.str()) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { diff --git a/src/statistics.cc b/src/statistics.cc index bd5a3d6597..c9235e0d3c 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -148,6 +148,7 @@ std::vector ComputeStats( // Get the data from the accumulator to BenchmarkReporter::Run's. Run data; data.run_name = reports[0].run_name; + data.family_index = reports[0].family_index; data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.threads = reports[0].threads; data.repetitions = reports[0].repetitions; diff --git a/test/BUILD b/test/BUILD index 9bb8cb02a0..1f27f99ede 100644 --- a/test/BUILD +++ b/test/BUILD @@ -20,6 +20,7 @@ TEST_ARGS = ["--benchmark_min_time=0.01"] PER_SRC_TEST_ARGS = ({ "user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"], + "repetitions_test.cc": [" --benchmark_repetitions=3"], }) load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 5681fdcf34..34731e8d85 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -13,7 +13,8 @@ namespace { int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) int AddComplexityTest(std::string test_name, std::string big_o_test_name, - std::string rms_test_name, std::string big_o) { + std::string rms_test_name, std::string big_o, + int family_index) { SetSubstitutions({{"%name", test_name}, {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, @@ -25,25 +26,29 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); - AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, - {"\"run_name\": \"%name\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"aggregate_name\": \"BigO\",$", MR_Next}, - {"\"cpu_coefficient\": %float,$", MR_Next}, - {"\"real_coefficient\": %float,$", MR_Next}, - {"\"big_o\": \"%bigo\",$", MR_Next}, - {"\"time_unit\": \"ns\"$", MR_Next}, - {"}", MR_Next}, - {"\"name\": \"%rms_name\",$"}, - {"\"run_name\": \"%name\",$", MR_Next}, - {"\"run_type\": \"aggregate\",$", MR_Next}, - {"\"repetitions\": %int,$", MR_Next}, - {"\"threads\": 1,$", MR_Next}, - {"\"aggregate_name\": \"RMS\",$", MR_Next}, - {"\"rms\": %float$", MR_Next}, - {"}", MR_Next}}); + AddCases( + TC_JSONOut, + {{"\"name\": \"%bigo_name\",$"}, + {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"BigO\",$", MR_Next}, + {"\"cpu_coefficient\": %float,$", MR_Next}, + {"\"real_coefficient\": %float,$", MR_Next}, + {"\"big_o\": \"%bigo\",$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}, + {"\"name\": \"%rms_name\",$"}, + {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": %int,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"RMS\",$", MR_Next}, + {"\"rms\": %float$", MR_Next}, + {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, {"^\"%bigo_name\"", MR_Not}, {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); @@ -82,15 +87,15 @@ const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_big_o_1); + enum_big_o_1, /*family_index=*/0); // Add auto enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - auto_big_o_1); + auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - lambda_big_o_1); + lambda_big_o_1, /*family_index=*/2); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -137,11 +142,11 @@ const char *lambda_big_o_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n); + enum_auto_big_o_n, /*family_index=*/3); // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n); + lambda_big_o_n, /*family_index=*/4); // ========================================================================= // // ------------------------- Testing BigO O(N*lgN) ------------------------- // @@ -178,11 +183,13 @@ const char *lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, + /*family_index=*/6); // Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + /*family_index=*/7); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -204,7 +211,7 @@ const std::string complexity_capture_name = "BM_ComplexityCaptureArgs/capture_test"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N"); + complexity_capture_name + "_RMS", "N", /*family_index=*/9); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/test/filter_test.cc b/test/filter_test.cc index fece6dc243..1c198913b3 100644 --- a/test/filter_test.cc +++ b/test/filter_test.cc @@ -1,15 +1,15 @@ -#include "benchmark/benchmark.h" - +#include #include #include #include #include - #include #include #include #include +#include "benchmark/benchmark.h" + namespace { class TestReporter : public benchmark::ConsoleReporter { @@ -20,17 +20,22 @@ class TestReporter : public benchmark::ConsoleReporter { virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { ++count_; + max_family_index_ = + std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); }; - TestReporter() : count_(0) {} + TestReporter() : count_(0), max_family_index_(0) {} virtual ~TestReporter() {} size_t GetCount() const { return count_; } + size_t GetMaxFamilyIndex() const { return max_family_index_; } + private: mutable size_t count_; + mutable size_t max_family_index_; }; } // end namespace @@ -98,6 +103,15 @@ int main(int argc, char **argv) { << std::endl; return -1; } + + const size_t max_family_index = test_reporter.GetMaxFamilyIndex(); + const size_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; + if (num_families != expected_reports) { + std::cerr << "ERROR: Expected " << expected_reports + << " test families to be run but num_families = " + << num_families << std::endl; + return -1; + } } return 0; diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 4e0e6495ba..0c312744f0 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -21,6 +21,7 @@ BENCHMARK(BM_empty); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc index 5980c99367..a5ab875914 100644 --- a/test/repetitions_test.cc +++ b/test/repetitions_test.cc @@ -24,6 +24,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_ExplicitRepetitions/repeats:2_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -36,6 +37,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -48,6 +50,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -60,6 +63,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -72,6 +76,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -108,6 +113,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_mean %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_median %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -119,6 +125,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -130,6 +137,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -141,6 +149,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -152,6 +161,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -163,6 +173,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 31bedfa7cd..91c6f497eb 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -71,6 +71,7 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -99,6 +100,7 @@ BENCHMARK(BM_bytes_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " "bytes_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -128,6 +130,7 @@ BENCHMARK(BM_items_per_second); ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " "items_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -154,6 +157,7 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -181,6 +185,7 @@ BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_nanosecond\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -202,6 +207,7 @@ BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_microsecond\",$"}, + {"\"family_index\": 5,$", MR_Next}, {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -223,6 +229,7 @@ BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_millisecond\",$"}, + {"\"family_index\": 6,$", MR_Next}, {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -243,6 +250,7 @@ BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"}, + {"\"family_index\": 7,$", MR_Next}, {"\"run_name\": \"BM_time_label_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -267,6 +275,7 @@ void BM_error(benchmark::State& state) { BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"family_index\": 8,$", MR_Next}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -289,6 +298,7 @@ void BM_no_arg_name(benchmark::State& state) { BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, + {"\"family_index\": 9,$", MR_Next}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -307,6 +317,7 @@ void BM_arg_name(benchmark::State& state) { BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, + {"\"family_index\": 10,$", MR_Next}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -327,6 +338,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, + {"\"family_index\": 11,$", MR_Next}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -346,6 +358,7 @@ BENCHMARK(BM_name)->Name("BM_custom_name"); ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"}, + {"\"family_index\": 12,$", MR_Next}, {"\"run_name\": \"BM_custom_name\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -405,18 +418,21 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -424,6 +440,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -431,6 +448,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, + {"\"family_index\": 15,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -452,24 +470,28 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, {"\"repetition_index\": 2,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -477,6 +499,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -484,6 +507,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, + {"\"family_index\": 16,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -507,30 +531,35 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 1,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 2,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, {"\"repetition_index\": 3,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -538,6 +567,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -545,6 +575,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, + {"\"family_index\": 17,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -568,6 +599,7 @@ void BM_RepeatOnce(benchmark::State& state) { BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, + {"\"family_index\": 18,$", MR_Next}, {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -590,6 +622,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -597,6 +630,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -604,6 +638,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, + {"\"family_index\": 19,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -632,6 +667,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -639,6 +675,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"mean\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -646,6 +683,7 @@ ADD_CASES(TC_JSONOut, {"\"aggregate_name\": \"median\",$", MR_Next}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, + {"\"family_index\": 20,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -678,6 +716,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -686,6 +725,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -694,6 +734,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"family_index\": 21,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -746,6 +787,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " ADD_CASES( TC_JSONOut, {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -755,6 +797,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -764,6 +807,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -773,6 +817,7 @@ ADD_CASES( {"\"iterations\": 5,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -782,6 +827,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -791,6 +837,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -800,6 +847,7 @@ ADD_CASES( {"\"iterations\": 3,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, + {"\"family_index\": 22,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -831,6 +879,7 @@ void BM_JSON_Format(benchmark::State& state) { } BENCHMARK(BM_JSON_Format); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, + {"\"family_index\": 23,$", MR_Next}, {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index d49ce55fbb..056923e3ba 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -71,6 +71,7 @@ void BM_Counters_Tabular(benchmark::State& state) { BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -123,6 +124,7 @@ void BM_CounterRates_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -174,6 +176,7 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -212,6 +215,7 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -254,6 +258,7 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 4d53f5c424..29b33f2d9b 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -32,6 +32,7 @@ BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -78,6 +79,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " "foo=%hrfloat items_per_second=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"family_index\": 1,$", MR_Next}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -125,6 +127,7 @@ ADD_CASES( TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"family_index\": 2,$", MR_Next}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -165,6 +168,7 @@ BENCHMARK(BM_Invert); ADD_CASES(TC_ConsoleOut, {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, + {"\"family_index\": 3,$", MR_Next}, {"\"run_name\": \"BM_Invert\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -207,6 +211,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " "bar=%hrfloats foo=%hrfloats$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_InvertedRate\",$"}, + {"\"family_index\": 4,$", MR_Next}, {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -246,6 +251,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"family_index\": 5,$", MR_Next}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -285,6 +291,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " "%console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"family_index\": 6,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -327,6 +334,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"family_index\": 7,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -367,6 +375,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"family_index\": 8,$", MR_Next}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -412,6 +421,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"family_index\": 9,$", MR_Next}, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -455,6 +465,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " "bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"family_index\": 10,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -498,6 +509,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"family_index\": 11,$", MR_Next}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 21d8285ded..842b36685c 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -51,6 +51,7 @@ ADD_CASES( }); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -68,6 +69,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -85,6 +87,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -102,6 +105,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -119,6 +123,7 @@ ADD_CASES(TC_JSONOut, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, From 80a62618e8a82b56492aecd80921af2166212783 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 2 Jun 2021 23:45:41 +0300 Subject: [PATCH 314/330] Introduce per-family instance index (#1165) Much like it makes sense to enumerate all the families, it makes sense to enumerate stuff within families. Alternatively, we could have a global instance index, but i'm not sure why that would be better. This will be useful when the benchmarks are run not in order, for the tools to sort the results properly. --- include/benchmark/benchmark.h | 1 + src/benchmark_api_internal.cc | 2 + src/benchmark_api_internal.h | 3 + src/benchmark_register.cc | 6 +- src/benchmark_runner.cc | 1 + src/complexity.cc | 2 + src/json_reporter.cc | 3 + src/statistics.cc | 1 + test/complexity_test.cc | 2 + test/memory_manager_test.cc | 1 + test/repetitions_test.cc | 11 ++ test/reporter_output_test.cc | 51 +++++- test/user_counters_tabular_test.cc | 248 +++++++++++++++++++++++++-- test/user_counters_test.cc | 12 ++ test/user_counters_thousands_test.cc | 5 + 15 files changed, 328 insertions(+), 21 deletions(-) diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 693c2b82f5..5b53debd25 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1419,6 +1419,7 @@ class BenchmarkReporter { std::string benchmark_name() const; BenchmarkName run_name; int64_t family_index; + int64_t per_family_instance_index; RunType run_type; std::string aggregate_name; std::string report_label; // Empty if not set by benchmark. diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index c4813172c5..89da519afc 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -8,10 +8,12 @@ namespace benchmark { namespace internal { BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, + int per_family_instance_idx, const std::vector& args, int thread_count) : benchmark_(*benchmark), family_index_(family_idx), + per_family_instance_index_(per_family_instance_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), time_unit_(benchmark_.time_unit_), diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 5ee6fadf04..e2afbd833c 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -18,10 +18,12 @@ namespace internal { class BenchmarkInstance { public: BenchmarkInstance(Benchmark* benchmark, int family_index, + int per_family_instance_index, const std::vector& args, int threads); const BenchmarkName& name() const { return name_; } int family_index() const { return family_index_; } + int per_family_instance_index() const { return per_family_instance_index_; } AggregationReportMode aggregation_report_mode() const { return aggregation_report_mode_; } @@ -47,6 +49,7 @@ class BenchmarkInstance { BenchmarkName name_; Benchmark& benchmark_; const int family_index_; + const int per_family_instance_index_; AggregationReportMode aggregation_report_mode_; const std::vector& args_; TimeUnit time_unit_; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index bc6c66869b..435c48b4aa 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -134,6 +134,7 @@ bool BenchmarkFamilies::FindBenchmarks( MutexLock l(mutex_); for (std::unique_ptr& family : families_) { int family_index = next_family_index; + int per_family_instance_index = 0; // Family was deleted or benchmark doesn't match if (!family) continue; @@ -158,7 +159,8 @@ bool BenchmarkFamilies::FindBenchmarks( for (auto const& args : family->args_) { for (int num_threads : *thread_counts) { - BenchmarkInstance instance(family.get(), family_index, args, + BenchmarkInstance instance(family.get(), family_index, + per_family_instance_index, args, num_threads); const auto full_name = instance.name().str(); @@ -167,6 +169,8 @@ bool BenchmarkFamilies::FindBenchmarks( instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); + ++per_family_instance_index; + // Only bump the next family index once we've estabilished that // at least one instance of this family will be run. if (next_family_index == family_index) ++next_family_index; diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index c71fc719dc..a9ff55c47d 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -73,6 +73,7 @@ BenchmarkReporter::Run CreateRunReport( report.run_name = b.name(); report.family_index = b.family_index(); + report.per_family_instance_index = b.per_family_instance_index(); report.error_occurred = results.has_error_; report.error_message = results.error_message_; report.report_label = results.report_label_; diff --git a/src/complexity.cc b/src/complexity.cc index 22f4263686..d74b146922 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -194,6 +194,7 @@ std::vector ComputeBigO( Run big_o; big_o.run_name = run_name; big_o.family_index = reports[0].family_index; + big_o.per_family_instance_index = reports[0].per_family_instance_index; big_o.run_type = BenchmarkReporter::Run::RT_Aggregate; big_o.repetitions = reports[0].repetitions; big_o.repetition_index = Run::no_repetition_index; @@ -217,6 +218,7 @@ std::vector ComputeBigO( Run rms; rms.run_name = run_name; rms.family_index = reports[0].family_index; + rms.per_family_instance_index = reports[0].per_family_instance_index; rms.run_type = BenchmarkReporter::Run::RT_Aggregate; rms.aggregate_name = "RMS"; rms.report_label = big_o.report_label; diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 5a7d843d3c..26898456f8 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -208,6 +208,9 @@ void JSONReporter::PrintRunData(Run const& run) { std::ostream& out = GetOutputStream(); out << indent << FormatKV("name", run.benchmark_name()) << ",\n"; out << indent << FormatKV("family_index", run.family_index) << ",\n"; + out << indent + << FormatKV("per_family_instance_index", run.per_family_instance_index) + << ",\n"; out << indent << FormatKV("run_name", run.run_name.str()) << ",\n"; out << indent << FormatKV("run_type", [&run]() -> const char* { switch (run.run_type) { diff --git a/src/statistics.cc b/src/statistics.cc index c9235e0d3c..57472b9ff9 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -149,6 +149,7 @@ std::vector ComputeStats( Run data; data.run_name = reports[0].run_name; data.family_index = reports[0].family_index; + data.per_family_instance_index = reports[0].per_family_instance_index; data.run_type = BenchmarkReporter::Run::RT_Aggregate; data.threads = reports[0].threads; data.repetitions = reports[0].repetitions; diff --git a/test/complexity_test.cc b/test/complexity_test.cc index 34731e8d85..0de73c5722 100644 --- a/test/complexity_test.cc +++ b/test/complexity_test.cc @@ -30,6 +30,7 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": %int,$", MR_Next}, @@ -42,6 +43,7 @@ int AddComplexityTest(std::string test_name, std::string big_o_test_name, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, {"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"%name\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": %int,$", MR_Next}, diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc index 0c312744f0..f0c192fcbd 100644 --- a/test/memory_manager_test.cc +++ b/test/memory_manager_test.cc @@ -22,6 +22,7 @@ BENCHMARK(BM_empty); ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_empty\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc index a5ab875914..f93de502a3 100644 --- a/test/repetitions_test.cc +++ b/test/repetitions_test.cc @@ -25,6 +25,7 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -38,6 +39,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -51,6 +53,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_mean\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -64,6 +67,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_median\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -77,6 +81,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ExplicitRepetitions/repeats:2_stddev\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ExplicitRepetitions/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -114,6 +119,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_median %console_report$"}}); ADD_CASES(TC_ConsoleOut, {{"^BM_ImplicitRepetitions_stddev %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -126,6 +132,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -138,6 +145,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -150,6 +158,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions\",$"}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -162,6 +171,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_mean\",$"}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -174,6 +184,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_median\",$"}, {"}", MR_Next}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_ImplicitRepetitions_stddev\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_ImplicitRepetitions\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index 91c6f497eb..989eb48ecc 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -72,6 +72,7 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_basic\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -101,6 +102,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " "bytes_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -131,6 +133,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " "items_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, {"\"family_index\": 2,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -158,6 +161,7 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, {"\"family_index\": 3,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_label\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -186,6 +190,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_nanosecond\",$"}, {"\"family_index\": 4,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -208,6 +213,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_microsecond\",$"}, {"\"family_index\": 5,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -230,6 +236,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_millisecond\",$"}, {"\"family_index\": 6,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -251,6 +258,7 @@ BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond); ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"}, {"\"family_index\": 7,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_time_label_second\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -276,6 +284,7 @@ BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, {"\"family_index\": 8,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_error\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -299,6 +308,7 @@ BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, {"\"family_index\": 9,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -318,6 +328,7 @@ BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, {"\"family_index\": 10,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -339,6 +350,7 @@ ADD_CASES(TC_ConsoleOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, {"\"family_index\": 11,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -359,6 +371,7 @@ BENCHMARK(BM_name)->Name("BM_custom_name"); ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"}, {"\"family_index\": 12,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_custom_name\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -419,6 +432,7 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"family_index\": 15,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -426,6 +440,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"family_index\": 15,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -433,6 +448,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, {"\"family_index\": 15,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -441,6 +457,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, {"\"family_index\": 15,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -449,6 +466,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, {"\"family_index\": 15,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -471,6 +489,7 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -478,6 +497,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -485,6 +505,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -492,6 +513,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -500,6 +522,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -508,6 +531,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, {"\"family_index\": 16,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -532,6 +556,7 @@ ADD_CASES(TC_ConsoleOut, {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -539,6 +564,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -546,6 +572,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -553,6 +580,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -560,6 +588,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"threads\": 1,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -568,6 +597,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -576,6 +606,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, {"\"family_index\": 17,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 4,$", MR_Next}, @@ -600,6 +631,7 @@ BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, {"\"family_index\": 18,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -623,6 +655,7 @@ ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, {"\"family_index\": 19,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -631,6 +664,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, {"\"family_index\": 19,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -639,6 +673,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, {"\"family_index\": 19,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -668,6 +703,7 @@ ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, {"\"family_index\": 20,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -676,6 +712,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, {"\"family_index\": 20,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -684,6 +721,7 @@ ADD_CASES(TC_JSONOut, {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, {"\"family_index\": 20,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -717,6 +755,7 @@ ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, {"\"family_index\": 21,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -726,6 +765,7 @@ ADD_CASES(TC_JSONOut, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, {"\"family_index\": 21,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -735,6 +775,7 @@ ADD_CASES(TC_JSONOut, {"\"time_unit\": \"us\",?$"}, {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, {"\"family_index\": 21,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 3,$", MR_Next}, @@ -788,6 +829,7 @@ ADD_CASES( TC_JSONOut, {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -798,6 +840,7 @@ ADD_CASES( {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -808,6 +851,7 @@ ADD_CASES( {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -818,6 +862,7 @@ ADD_CASES( {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -828,6 +873,7 @@ ADD_CASES( {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -838,6 +884,7 @@ ADD_CASES( {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -848,6 +895,7 @@ ADD_CASES( {"\"real_time\": %float,$", MR_Next}, {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, {"\"family_index\": 22,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, @@ -879,7 +927,8 @@ void BM_JSON_Format(benchmark::State& state) { } BENCHMARK(BM_JSON_Format); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"}, - {"\"family_index\": 23,$", MR_Next}, + {"\"family_index\": 23,$", MR_Next}, +{"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_JSON_Format\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc index 056923e3ba..421f27b5cb 100644 --- a/test/user_counters_tabular_test.cc +++ b/test/user_counters_tabular_test.cc @@ -7,19 +7,23 @@ // @todo: this checks the full output at once; the rule for // CounterSet1 was failing because it was not matching "^[-]+$". // @todo: check that the counters are vertically aligned. -ADD_CASES( - TC_ConsoleOut, - { - // keeping these lines long improves readability, so: - // clang-format off +ADD_CASES(TC_ConsoleOut, + { + // keeping these lines long improves readability, so: + // clang-format off {"^[-]+$", MR_Next}, {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, {"^[-]+$", MR_Next}, - {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, - {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, - {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, - {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, - {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:1 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:1_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:1_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:1_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:2 %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:2_mean %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:2_median %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/repeats:2/threads:2_stddev %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, @@ -46,8 +50,8 @@ ADD_CASES( {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}, - // clang-format on - }); + // clang-format on + }); ADD_CASES(TC_CSVOut, {{"%csv_header," "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); @@ -68,13 +72,15 @@ void BM_Counters_Tabular(benchmark::State& state) { {"Lob", {32, bm::Counter::kAvgThreads}}, }); } -BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); +BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 2)->Repetitions(2); ADD_CASES(TC_JSONOut, - {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"}, {"\"family_index\": 0,$", MR_Next}, - {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", + MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, - {"\"repetitions\": 1,$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, {"\"repetition_index\": 0,$", MR_Next}, {"\"threads\": 1,$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, @@ -88,8 +94,205 @@ ADD_CASES(TC_JSONOut, {"\"Frob\": %float,$", MR_Next}, {"\"Lob\": %float$", MR_Next}, {"}", MR_Next}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," - "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:1_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:1\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 1,$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); + +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 1,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 2,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 1,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"repetition_index\": 1,$", MR_Next}, + {"\"threads\": 2,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 1,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 2,$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/repeats:2/threads:2_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 1,$", MR_Next}, + {"\"run_name\": \"BM_Counters_Tabular/repeats:2/threads:2\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 2,$", MR_Next}, + {"\"threads\": 2,$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:1\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_mean\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_median\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_mean\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_median\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() void CheckTabular(Results const& e) { @@ -100,7 +303,10 @@ void CheckTabular(Results const& e) { CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32); } -CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); +CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:1$", + &CheckTabular); +CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$", + &CheckTabular); // ========================================================================= // // -------------------- Tabular+Rate Counters Output ----------------------- // @@ -125,6 +331,7 @@ BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -177,6 +384,7 @@ BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, {"\"family_index\": 2,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -216,6 +424,7 @@ BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, {"\"family_index\": 3,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -259,6 +468,7 @@ BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, {"\"family_index\": 4,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc index 29b33f2d9b..377bb32ca9 100644 --- a/test/user_counters_test.cc +++ b/test/user_counters_test.cc @@ -33,6 +33,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -80,6 +81,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, {"\"family_index\": 1,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -128,6 +130,7 @@ ADD_CASES( {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, {"\"family_index\": 2,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -169,6 +172,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"}, {"\"family_index\": 3,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Invert\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -212,6 +216,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_InvertedRate\",$"}, {"\"family_index\": 4,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -252,6 +257,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, {"\"family_index\": 5,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -292,6 +298,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, {"\"family_index\": 6,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -335,6 +342,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, {"\"family_index\": 7,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -376,6 +384,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, {"\"family_index\": 8,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -422,6 +431,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, {"\"family_index\": 9,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, @@ -466,6 +476,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, {"\"family_index\": 10,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, @@ -510,6 +521,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, {"\"family_index\": 11,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 1,$", MR_Next}, diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc index 842b36685c..bbe194264e 100644 --- a/test/user_counters_thousands_test.cc +++ b/test/user_counters_thousands_test.cc @@ -52,6 +52,7 @@ ADD_CASES( ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -70,6 +71,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"iteration\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -88,6 +90,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -106,6 +109,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, @@ -124,6 +128,7 @@ ADD_CASES(TC_JSONOut, ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, {"\"run_type\": \"aggregate\",$", MR_Next}, {"\"repetitions\": 2,$", MR_Next}, From 0c1da0a713c6d9771065aa85d29958e14780d63b Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 3 Jun 2021 11:46:34 +0300 Subject: [PATCH 315/330] Make 'complexity reports' cache per-family, not global (#1166) While the current variant works, it assumes that all the instances of a single family will be run together, with nothing inbetween them. Naturally, that won't work once the runs may be interleaved. --- src/benchmark.cc | 16 ++++++++++++---- src/benchmark_runner.cc | 19 ++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index d205232b8a..cf24ffdc91 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -265,8 +265,9 @@ void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter::Context context; context.name_field_width = name_field_width; - // Keep track of running times of all instances of current benchmark - std::vector complexity_reports; + // Keep track of running times of all instances of each benchmark family. + std::map> + complexity_reports; // We flush streams after invoking reporter methods that write to them. This // ensures users get timely updates even when streams are not line-buffered. @@ -281,8 +282,15 @@ void RunBenchmarks(const std::vector& benchmarks, flushStreams(display_reporter); flushStreams(file_reporter); - for (const auto& benchmark : benchmarks) { - RunResults run_results = RunBenchmark(benchmark, &complexity_reports); + for (const BenchmarkInstance& benchmark : benchmarks) { + std::vector* complexity_reports_for_family = + nullptr; + if (benchmark.complexity() != oNone) + complexity_reports_for_family = + &complexity_reports[benchmark.family_index()]; + + RunResults run_results = + RunBenchmark(benchmark, complexity_reports_for_family); auto report = [&run_results](BenchmarkReporter* reporter, bool report_aggregates_only) { diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index a9ff55c47d..4869aa7b27 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -142,10 +142,11 @@ class BenchmarkRunner { BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, std::vector* complexity_reports_) : b(b_), - complexity_reports(*complexity_reports_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + complexity_reports(complexity_reports_), + min_time(!IsZero(b.min_time()) ? b.min_time() + : FLAGS_benchmark_min_time), repeats(b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), + : FLAGS_benchmark_repetitions), has_explicit_iteration_count(b.iterations() != 0), pool(b.threads() - 1), iters(has_explicit_iteration_count ? b.iterations() : 1), @@ -178,12 +179,12 @@ class BenchmarkRunner { run_results.aggregates_only = ComputeStats(run_results.non_aggregates); // Maybe calculate complexity report - if ((b.complexity() != oNone) && b.last_benchmark_instance) { - auto additional_run_stats = ComputeBigO(complexity_reports); + if (complexity_reports && b.last_benchmark_instance) { + auto additional_run_stats = ComputeBigO(*complexity_reports); run_results.aggregates_only.insert(run_results.aggregates_only.end(), additional_run_stats.begin(), additional_run_stats.end()); - complexity_reports.clear(); + complexity_reports->clear(); } } @@ -193,7 +194,7 @@ class BenchmarkRunner { RunResults run_results; const benchmark::internal::BenchmarkInstance& b; - std::vector& complexity_reports; + std::vector* complexity_reports; const double min_time; const int repeats; @@ -360,8 +361,8 @@ class BenchmarkRunner { CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, repetition_index, repeats); - if (!report.error_occurred && b.complexity() != oNone) - complexity_reports.push_back(report); + if (complexity_reports && !report.error_occurred) + complexity_reports->push_back(report); run_results.non_aggregates.push_back(report); } From 6e32352c79dac230861bf601a66fb53815efa864 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 3 Jun 2021 16:22:52 +0300 Subject: [PATCH 316/330] compare.py: sort the results (#1168) Currently, the tooling just keeps the whatever benchmark order that was present, and this is fine nowadays, but once the benchmarks will be optionally run interleaved, that will be rather suboptimal. So, now that i have introduced family index and per-family instance index, we can define an order for the benchmarks, and sort them accordingly. There is a caveat with aggregates, we assume that they are in-order, and hopefully we won't mess that order up.. --- tools/compare.py | 8 +-- tools/gbench/Inputs/test4_run.json | 96 ++++++++++++++++++++++++++++++ tools/gbench/report.py | 47 ++++++++++++++- tools/gbench/util.py | 18 ++++++ 4 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 tools/gbench/Inputs/test4_run.json diff --git a/tools/compare.py b/tools/compare.py index 66eed932c2..01d2c89f50 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -238,10 +238,10 @@ def main(): options_contender = ['--benchmark_filter=%s' % filter_contender] # Run the benchmarks and report the results - json1 = json1_orig = gbench.util.run_or_load_benchmark( - test_baseline, benchmark_options + options_baseline) - json2 = json2_orig = gbench.util.run_or_load_benchmark( - test_contender, benchmark_options + options_contender) + json1 = json1_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark( + test_baseline, benchmark_options + options_baseline)) + json2 = json2_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark( + test_contender, benchmark_options + options_contender)) # Now, filter the benchmarks so that the difference report can work if filter_baseline and filter_contender: diff --git a/tools/gbench/Inputs/test4_run.json b/tools/gbench/Inputs/test4_run.json new file mode 100644 index 0000000000..eaa005f3a9 --- /dev/null +++ b/tools/gbench/Inputs/test4_run.json @@ -0,0 +1,96 @@ +{ + "benchmarks": [ + { + "name": "99 family 0 instance 0 repetition 0", + "run_type": "iteration", + "family_index": 0, + "per_family_instance_index": 0, + "repetition_index": 0 + }, + { + "name": "98 family 0 instance 0 repetition 1", + "run_type": "iteration", + "family_index": 0, + "per_family_instance_index": 0, + "repetition_index": 1 + }, + { + "name": "97 family 0 instance 0 aggregate", + "run_type": "aggregate", + "family_index": 0, + "per_family_instance_index": 0, + "aggregate_name": "9 aggregate" + }, + + + { + "name": "96 family 0 instance 1 repetition 0", + "run_type": "iteration", + "family_index": 0, + "per_family_instance_index": 1, + "repetition_index": 0 + }, + { + "name": "95 family 0 instance 1 repetition 1", + "run_type": "iteration", + "family_index": 0, + "per_family_instance_index": 1, + "repetition_index": 1 + }, + { + "name": "94 family 0 instance 1 aggregate", + "run_type": "aggregate", + "family_index": 0, + "per_family_instance_index": 1, + "aggregate_name": "9 aggregate" + }, + + + + + { + "name": "93 family 1 instance 0 repetition 0", + "run_type": "iteration", + "family_index": 1, + "per_family_instance_index": 0, + "repetition_index": 0 + }, + { + "name": "92 family 1 instance 0 repetition 1", + "run_type": "iteration", + "family_index": 1, + "per_family_instance_index": 0, + "repetition_index": 1 + }, + { + "name": "91 family 1 instance 0 aggregate", + "run_type": "aggregate", + "family_index": 1, + "per_family_instance_index": 0, + "aggregate_name": "9 aggregate" + }, + + + { + "name": "90 family 1 instance 1 repetition 0", + "run_type": "iteration", + "family_index": 1, + "per_family_instance_index": 1, + "repetition_index": 0 + }, + { + "name": "89 family 1 instance 1 repetition 1", + "run_type": "iteration", + "family_index": 1, + "per_family_instance_index": 1, + "repetition_index": 1 + }, + { + "name": "88 family 1 instance 1 aggregate", + "run_type": "aggregate", + "family_index": 1, + "per_family_instance_index": 1, + "aggregate_name": "9 aggregate" + } + ] +} diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 69b643e92b..6bea82f6bf 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -1,9 +1,11 @@ -import unittest """report.py - Utilities for reporting statistics about benchmark results """ + +import unittest import os import re import copy +import random from scipy.stats import mannwhitneyu @@ -912,6 +914,49 @@ def test_json_diff_report(self): assert_measurements(self, out, expected) +class TestReportSorting(unittest.TestCase): + @classmethod + def setUpClass(cls): + def load_result(): + import json + testInputs = os.path.join( + os.path.dirname( + os.path.realpath(__file__)), + 'Inputs') + testOutput = os.path.join(testInputs, 'test4_run.json') + with open(testOutput, 'r') as f: + json = json.load(f) + return json + + cls.json = load_result() + + def test_json_diff_report_pretty_printing(self): + import util + + expected_names = [ + "99 family 0 instance 0 repetition 0", + "98 family 0 instance 0 repetition 1", + "97 family 0 instance 0 aggregate", + "96 family 0 instance 1 repetition 0", + "95 family 0 instance 1 repetition 1", + "94 family 0 instance 1 aggregate", + "93 family 1 instance 0 repetition 0", + "92 family 1 instance 0 repetition 1", + "91 family 1 instance 0 aggregate", + "90 family 1 instance 1 repetition 0", + "89 family 1 instance 1 repetition 1", + "88 family 1 instance 1 aggregate" + ] + + for n in range(len(self.json['benchmarks']) ** 2): + random.shuffle(self.json['benchmarks']) + sorted_benchmarks = util.sort_benchmark_results(self.json)[ + 'benchmarks'] + self.assertEqual(len(expected_names), len(sorted_benchmarks)) + for out, expected in zip(sorted_benchmarks, expected_names): + self.assertEqual(out['name'], expected) + + def assert_utest(unittest_instance, lhs, rhs): if lhs['utest']: unittest_instance.assertAlmostEqual( diff --git a/tools/gbench/util.py b/tools/gbench/util.py index 661c4bad8d..5d0012c0cb 100644 --- a/tools/gbench/util.py +++ b/tools/gbench/util.py @@ -5,6 +5,7 @@ import tempfile import subprocess import sys +import functools # Input file type enumeration IT_Invalid = 0 @@ -119,6 +120,23 @@ def load_benchmark_results(fname): return json.load(f) +def sort_benchmark_results(result): + benchmarks = result['benchmarks'] + + # From inner key to the outer key! + benchmarks = sorted( + benchmarks, key=lambda benchmark: benchmark['repetition_index'] if 'repetition_index' in benchmark else -1) + benchmarks = sorted( + benchmarks, key=lambda benchmark: 1 if 'run_type' in benchmark and benchmark['run_type'] == "aggregate" else 0) + benchmarks = sorted( + benchmarks, key=lambda benchmark: benchmark['per_family_instance_index'] if 'per_family_instance_index' in benchmark else -1) + benchmarks = sorted( + benchmarks, key=lambda benchmark: benchmark['family_index'] if 'family_index' in benchmark else -1) + + result['benchmarks'] = benchmarks + return result + + def run_benchmark(exe_name, benchmark_flags): """ Run a benchmark specified by 'exe_name' with the specified From 520573fecb754f470c57f40a3753c84a6991d976 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 3 Jun 2021 16:42:08 +0300 Subject: [PATCH 317/330] [NFCI] RunBenchmarks(): extract FlushStreams()/Report() functions Based on original implementation by Hai Huang @haih-g in https://github.com/google/benchmark/pull/1105 --- src/benchmark.cc | 65 +++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index cf24ffdc91..443ffbf9fa 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -241,6 +241,37 @@ void State::FinishKeepRunning() { namespace internal { namespace { +// Flushes streams after invoking reporter methods that write to them. This +// ensures users get timely updates even when streams are not line-buffered. +void FlushStreams(BenchmarkReporter* reporter) { + if (!reporter) return; + std::flush(reporter->GetOutputStream()); + std::flush(reporter->GetErrorStream()); +} + +// Reports in both display and file reporters. +void Report(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, const RunResults& run_results) { + auto report_one = [](BenchmarkReporter* reporter, bool aggregates_only, + const RunResults& results) { + assert(reporter); + // If there are no aggregates, do output non-aggregates. + aggregates_only &= !results.aggregates_only.empty(); + if (!aggregates_only) reporter->ReportRuns(results.non_aggregates); + if (!results.aggregates_only.empty()) + reporter->ReportRuns(results.aggregates_only); + }; + + report_one(display_reporter, run_results.display_report_aggregates_only, + run_results); + if (file_reporter) + report_one(file_reporter, run_results.file_report_aggregates_only, + run_results); + + FlushStreams(display_reporter); + FlushStreams(file_reporter); +} + void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { @@ -269,18 +300,10 @@ void RunBenchmarks(const std::vector& benchmarks, std::map> complexity_reports; - // We flush streams after invoking reporter methods that write to them. This - // ensures users get timely updates even when streams are not line-buffered. - auto flushStreams = [](BenchmarkReporter* reporter) { - if (!reporter) return; - std::flush(reporter->GetOutputStream()); - std::flush(reporter->GetErrorStream()); - }; - if (display_reporter->ReportContext(context) && (!file_reporter || file_reporter->ReportContext(context))) { - flushStreams(display_reporter); - flushStreams(file_reporter); + FlushStreams(display_reporter); + FlushStreams(file_reporter); for (const BenchmarkInstance& benchmark : benchmarks) { std::vector* complexity_reports_for_family = @@ -292,29 +315,13 @@ void RunBenchmarks(const std::vector& benchmarks, RunResults run_results = RunBenchmark(benchmark, complexity_reports_for_family); - auto report = [&run_results](BenchmarkReporter* reporter, - bool report_aggregates_only) { - assert(reporter); - // If there are no aggregates, do output non-aggregates. - report_aggregates_only &= !run_results.aggregates_only.empty(); - if (!report_aggregates_only) - reporter->ReportRuns(run_results.non_aggregates); - if (!run_results.aggregates_only.empty()) - reporter->ReportRuns(run_results.aggregates_only); - }; - - report(display_reporter, run_results.display_report_aggregates_only); - if (file_reporter) - report(file_reporter, run_results.file_report_aggregates_only); - - flushStreams(display_reporter); - flushStreams(file_reporter); + Report(display_reporter, file_reporter, run_results); } } display_reporter->Finalize(); if (file_reporter) file_reporter->Finalize(); - flushStreams(display_reporter); - flushStreams(file_reporter); + FlushStreams(display_reporter); + FlushStreams(file_reporter); } // Disable deprecated warnings temporarily because we need to reference From 32cc60710761039d545612ecfa70b671a4d7e58e Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 3 Jun 2021 16:54:06 +0300 Subject: [PATCH 318/330] [NFCI] Make BenchmarkRunner non-internal to it's .cpp file Currently the lifetime of a single BenchmarkRunner is constrained to a RunBenchmark(), but that will have to change for interleaved benchmark execution, because we'll need to keep it around to not forget how much repetitions of an instance we've done. --- src/benchmark_runner.cc | 387 +++++++++++++++++++--------------------- src/benchmark_runner.h | 45 +++++ 2 files changed, 226 insertions(+), 206 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 4869aa7b27..55d6cf15d2 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "benchmark_runner.h" + #include "benchmark/benchmark.h" #include "benchmark_api_internal.h" #include "internal_macros.h" @@ -106,7 +107,8 @@ BenchmarkReporter::Run CreateRunReport( report.max_bytes_used = memory_result.max_bytes_used; } - internal::Finish(&report.counters, results.iterations, seconds, b.threads()); + internal::Finish(&report.counters, results.iterations, seconds, + b.threads()); } return report; } @@ -137,238 +139,211 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } -class BenchmarkRunner { - public: - BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, - std::vector* complexity_reports_) - : b(b_), - complexity_reports(complexity_reports_), - min_time(!IsZero(b.min_time()) ? b.min_time() - : FLAGS_benchmark_min_time), - repeats(b.repetitions() != 0 ? b.repetitions() - : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0), - pool(b.threads() - 1), - iters(has_explicit_iteration_count ? b.iterations() : 1), - perf_counters_measurement( - PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), - perf_counters_measurement_ptr(perf_counters_measurement.IsValid() - ? &perf_counters_measurement - : nullptr) { +} // end namespace + +BenchmarkRunner::BenchmarkRunner( + const benchmark::internal::BenchmarkInstance& b_, + std::vector* complexity_reports_) + : b(b_), + complexity_reports(complexity_reports_), + min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + repeats(b.repetitions() != 0 ? b.repetitions() + : FLAGS_benchmark_repetitions), + has_explicit_iteration_count(b.iterations() != 0), + pool(b.threads() - 1), + iters(has_explicit_iteration_count ? b.iterations() : 1), + perf_counters_measurement( + PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), + perf_counters_measurement_ptr(perf_counters_measurement.IsValid() + ? &perf_counters_measurement + : nullptr) { + run_results.display_report_aggregates_only = + (FLAGS_benchmark_report_aggregates_only || + FLAGS_benchmark_display_aggregates_only); + run_results.file_report_aggregates_only = + FLAGS_benchmark_report_aggregates_only; + if (b.aggregation_report_mode() != internal::ARM_Unspecified) { run_results.display_report_aggregates_only = - (FLAGS_benchmark_report_aggregates_only || - FLAGS_benchmark_display_aggregates_only); + (b.aggregation_report_mode() & + internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = - FLAGS_benchmark_report_aggregates_only; - if (b.aggregation_report_mode() != internal::ARM_Unspecified) { - run_results.display_report_aggregates_only = - (b.aggregation_report_mode() & - internal::ARM_DisplayReportAggregatesOnly); - run_results.file_report_aggregates_only = - (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); - CHECK(FLAGS_benchmark_perf_counters.empty() || - perf_counters_measurement.IsValid()) - << "Perf counters were requested but could not be set up."; - } - - for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { - DoOneRepetition(repetition_num); - } - - // Calculate additional statistics - run_results.aggregates_only = ComputeStats(run_results.non_aggregates); - - // Maybe calculate complexity report - if (complexity_reports && b.last_benchmark_instance) { - auto additional_run_stats = ComputeBigO(*complexity_reports); - run_results.aggregates_only.insert(run_results.aggregates_only.end(), - additional_run_stats.begin(), - additional_run_stats.end()); - complexity_reports->clear(); - } + (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); + CHECK(FLAGS_benchmark_perf_counters.empty() || + perf_counters_measurement.IsValid()) + << "Perf counters were requested but could not be set up."; } - RunResults&& get_results() { return std::move(run_results); } - - private: - RunResults run_results; + for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { + DoOneRepetition(repetition_num); + } - const benchmark::internal::BenchmarkInstance& b; - std::vector* complexity_reports; + // Calculate additional statistics + run_results.aggregates_only = ComputeStats(run_results.non_aggregates); - const double min_time; - const int repeats; - const bool has_explicit_iteration_count; + // Maybe calculate complexity report + if (complexity_reports && b.last_benchmark_instance) { + auto additional_run_stats = ComputeBigO(*complexity_reports); + run_results.aggregates_only.insert(run_results.aggregates_only.end(), + additional_run_stats.begin(), + additional_run_stats.end()); + complexity_reports->clear(); + } +} - std::vector pool; +BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { + VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; - IterationCount iters; // preserved between repetitions! - // So only the first repetition has to find/calculate it, - // the other repetitions will just use that precomputed iteration count. + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(b.threads())); - PerfCountersMeasurement perf_counters_measurement; - PerfCountersMeasurement* const perf_counters_measurement_ptr; + // Run all but one thread in separate threads + for (std::size_t ti = 0; ti < pool.size(); ++ti) { + pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), + manager.get(), perf_counters_measurement_ptr); + } + // And run one thread here directly. + // (If we were asked to run just one thread, we don't create new threads.) + // Yes, we need to do this here *after* we start the separate threads. + RunInThread(&b, iters, 0, manager.get(), perf_counters_measurement_ptr); - struct IterationResults { - internal::ThreadManager::Result results; - IterationCount iters; - double seconds; - }; - IterationResults DoNIterations() { - VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; + // The main thread has finished. Now let's wait for the other threads. + manager->WaitForAllThreads(); + for (std::thread& thread : pool) thread.join(); - std::unique_ptr manager; - manager.reset(new internal::ThreadManager(b.threads())); + IterationResults i; + // Acquire the measurements/counters from the manager, UNDER THE LOCK! + { + MutexLock l(manager->GetBenchmarkMutex()); + i.results = manager->results; + } - // Run all but one thread in separate threads - for (std::size_t ti = 0; ti < pool.size(); ++ti) { - pool[ti] = std::thread(&RunInThread, &b, iters, static_cast(ti + 1), - manager.get(), perf_counters_measurement_ptr); - } - // And run one thread here directly. - // (If we were asked to run just one thread, we don't create new threads.) - // Yes, we need to do this here *after* we start the separate threads. - RunInThread(&b, iters, 0, manager.get(), perf_counters_measurement_ptr); + // And get rid of the manager. + manager.reset(); - // The main thread has finished. Now let's wait for the other threads. - manager->WaitForAllThreads(); - for (std::thread& thread : pool) thread.join(); + // Adjust real/manual time stats since they were reported per thread. + i.results.real_time_used /= b.threads(); + i.results.manual_time_used /= b.threads(); + // If we were measuring whole-process CPU usage, adjust the CPU time too. + if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); - IterationResults i; - // Acquire the measurements/counters from the manager, UNDER THE LOCK! - { - MutexLock l(manager->GetBenchmarkMutex()); - i.results = manager->results; - } + VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" + << i.results.real_time_used << "\n"; - // And get rid of the manager. - manager.reset(); + // By using KeepRunningBatch a benchmark can iterate more times than + // requested, so take the iteration count from i.results. + i.iters = i.results.iterations / b.threads(); - // Adjust real/manual time stats since they were reported per thread. - i.results.real_time_used /= b.threads(); - i.results.manual_time_used /= b.threads(); - // If we were measuring whole-process CPU usage, adjust the CPU time too. - if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); + // Base decisions off of real time if requested by this benchmark. + i.seconds = i.results.cpu_time_used; + if (b.use_manual_time()) { + i.seconds = i.results.manual_time_used; + } else if (b.use_real_time()) { + i.seconds = i.results.real_time_used; + } - VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" - << i.results.real_time_used << "\n"; + return i; +} - // By using KeepRunningBatch a benchmark can iterate more times than - // requested, so take the iteration count from i.results. - i.iters = i.results.iterations / b.threads(); +IterationCount BenchmarkRunner::PredictNumItersNeeded( + const IterationResults& i) const { + // See how much iterations should be increased by. + // Note: Avoid division by zero with max(seconds, 1ns). + double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); + // If our last run was at least 10% of FLAGS_benchmark_min_time then we + // use the multiplier directly. + // Otherwise we use at most 10 times expansion. + // NOTE: When the last run was at least 10% of the min time the max + // expansion should be 14x. + bool is_significant = (i.seconds / min_time) > 0.1; + multiplier = is_significant ? multiplier : std::min(10.0, multiplier); + if (multiplier <= 1.0) multiplier = 2.0; + + // So what seems to be the sufficiently-large iteration count? Round up. + const IterationCount max_next_iters = static_cast( + std::lround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); + // But we do have *some* sanity limits though.. + const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); + + VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; + return next_iters; // round up before conversion to integer. +} - // Base decisions off of real time if requested by this benchmark. - i.seconds = i.results.cpu_time_used; - if (b.use_manual_time()) { - i.seconds = i.results.manual_time_used; - } else if (b.use_real_time()) { - i.seconds = i.results.real_time_used; - } +bool BenchmarkRunner::ShouldReportIterationResults( + const IterationResults& i) const { + // Determine if this run should be reported; + // Either it has run for a sufficient amount of time + // or because an error was reported. + return i.results.has_error_ || + i.iters >= kMaxIterations || // Too many iterations already. + i.seconds >= min_time || // The elapsed time is large enough. + // CPU time is specified but the elapsed real time greatly exceeds + // the minimum time. + // Note that user provided timers are except from this sanity check. + ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); +} - return i; +void BenchmarkRunner::DoOneRepetition(int64_t repetition_index) { + const bool is_the_first_repetition = repetition_index == 0; + IterationResults i; + + // We *may* be gradually increasing the length (iteration count) + // of the benchmark until we decide the results are significant. + // And once we do, we report those last results and exit. + // Please do note that the if there are repetitions, the iteration count + // is *only* calculated for the *first* repetition, and other repetitions + // simply use that precomputed iteration count. + for (;;) { + i = DoNIterations(); + + // Do we consider the results to be significant? + // If we are doing repetitions, and the first repetition was already done, + // it has calculated the correct iteration time, so we have run that very + // iteration count just now. No need to calculate anything. Just report. + // Else, the normal rules apply. + const bool results_are_significant = !is_the_first_repetition || + has_explicit_iteration_count || + ShouldReportIterationResults(i); + + if (results_are_significant) break; // Good, let's report them! + + // Nope, bad iteration. Let's re-estimate the hopefully-sufficient + // iteration count, and run the benchmark again... + + iters = PredictNumItersNeeded(i); + assert(iters > i.iters && + "if we did more iterations than we want to do the next time, " + "then we should have accepted the current iteration run."); } - IterationCount PredictNumItersNeeded(const IterationResults& i) const { - // See how much iterations should be increased by. - // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); - // If our last run was at least 10% of FLAGS_benchmark_min_time then we - // use the multiplier directly. - // Otherwise we use at most 10 times expansion. - // NOTE: When the last run was at least 10% of the min time the max - // expansion should be 14x. - bool is_significant = (i.seconds / min_time) > 0.1; - multiplier = is_significant ? multiplier : std::min(10.0, multiplier); - if (multiplier <= 1.0) multiplier = 2.0; - - // So what seems to be the sufficiently-large iteration count? Round up. - const IterationCount max_next_iters = static_cast( - std::lround(std::max(multiplier * static_cast(i.iters), - static_cast(i.iters) + 1.0))); - // But we do have *some* sanity limits though.. - const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); - - VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; - return next_iters; // round up before conversion to integer. - } + // Oh, one last thing, we need to also produce the 'memory measurements'.. + MemoryManager::Result memory_result; + IterationCount memory_iterations = 0; + if (memory_manager != nullptr) { + // Only run a few iterations to reduce the impact of one-time + // allocations in benchmarks that are not properly managed. + memory_iterations = std::min(16, iters); + memory_manager->Start(); + std::unique_ptr manager; + manager.reset(new internal::ThreadManager(1)); + RunInThread(&b, memory_iterations, 0, manager.get(), + perf_counters_measurement_ptr); + manager->WaitForAllThreads(); + manager.reset(); - bool ShouldReportIterationResults(const IterationResults& i) const { - // Determine if this run should be reported; - // Either it has run for a sufficient amount of time - // or because an error was reported. - return i.results.has_error_ || - i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= min_time || // The elapsed time is large enough. - // CPU time is specified but the elapsed real time greatly exceeds - // the minimum time. - // Note that user provided timers are except from this sanity check. - ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); + memory_manager->Stop(&memory_result); } - void DoOneRepetition(int64_t repetition_index) { - const bool is_the_first_repetition = repetition_index == 0; - IterationResults i; - - // We *may* be gradually increasing the length (iteration count) - // of the benchmark until we decide the results are significant. - // And once we do, we report those last results and exit. - // Please do note that the if there are repetitions, the iteration count - // is *only* calculated for the *first* repetition, and other repetitions - // simply use that precomputed iteration count. - for (;;) { - i = DoNIterations(); - - // Do we consider the results to be significant? - // If we are doing repetitions, and the first repetition was already done, - // it has calculated the correct iteration time, so we have run that very - // iteration count just now. No need to calculate anything. Just report. - // Else, the normal rules apply. - const bool results_are_significant = !is_the_first_repetition || - has_explicit_iteration_count || - ShouldReportIterationResults(i); - - if (results_are_significant) break; // Good, let's report them! - - // Nope, bad iteration. Let's re-estimate the hopefully-sufficient - // iteration count, and run the benchmark again... - - iters = PredictNumItersNeeded(i); - assert(iters > i.iters && - "if we did more iterations than we want to do the next time, " - "then we should have accepted the current iteration run."); - } - - // Oh, one last thing, we need to also produce the 'memory measurements'.. - MemoryManager::Result memory_result; - IterationCount memory_iterations = 0; - if (memory_manager != nullptr) { - // Only run a few iterations to reduce the impact of one-time - // allocations in benchmarks that are not properly managed. - memory_iterations = std::min(16, iters); - memory_manager->Start(); - std::unique_ptr manager; - manager.reset(new internal::ThreadManager(1)); - RunInThread(&b, memory_iterations, 0, manager.get(), - perf_counters_measurement_ptr); - manager->WaitForAllThreads(); - manager.reset(); - - memory_manager->Stop(&memory_result); - } - - // Ok, now actualy report. - BenchmarkReporter::Run report = - CreateRunReport(b, i.results, memory_iterations, memory_result, - i.seconds, repetition_index, repeats); - - if (complexity_reports && !report.error_occurred) - complexity_reports->push_back(report); + // Ok, now actualy report. + BenchmarkReporter::Run report = + CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, + repetition_index, repeats); - run_results.non_aggregates.push_back(report); - } -}; + if (complexity_reports && !report.error_occurred) + complexity_reports->push_back(report); -} // end namespace + run_results.non_aggregates.push_back(report); +} RunResults RunBenchmark( const benchmark::internal::BenchmarkInstance& b, diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 9b0cf2a64e..9730ad386c 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -15,8 +15,13 @@ #ifndef BENCHMARK_RUNNER_H_ #define BENCHMARK_RUNNER_H_ +#include +#include + #include "benchmark_api_internal.h" #include "internal_macros.h" +#include "perf_counters.h" +#include "thread_manager.h" DECLARE_double(benchmark_min_time); @@ -42,6 +47,46 @@ struct RunResults { bool file_report_aggregates_only = false; }; +class BenchmarkRunner { + public: + BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, + std::vector* complexity_reports_); + + RunResults&& get_results() { return std::move(run_results); } + + private: + RunResults run_results; + + const benchmark::internal::BenchmarkInstance& b; + std::vector* complexity_reports; + + const double min_time; + const int repeats; + const bool has_explicit_iteration_count; + + std::vector pool; + + IterationCount iters; // preserved between repetitions! + // So only the first repetition has to find/calculate it, + // the other repetitions will just use that precomputed iteration count. + + PerfCountersMeasurement perf_counters_measurement; + PerfCountersMeasurement* const perf_counters_measurement_ptr; + + struct IterationResults { + internal::ThreadManager::Result results; + IterationCount iters; + double seconds; + }; + IterationResults DoNIterations(); + + IterationCount PredictNumItersNeeded(const IterationResults& i) const; + + bool ShouldReportIterationResults(const IterationResults& i) const; + + void DoOneRepetition(int64_t repetition_index); +}; + RunResults RunBenchmark( const benchmark::internal::BenchmarkInstance& b, std::vector* complexity_reports); From d17ea665515f0c54d100c6fc973632431379f64b Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 3 Jun 2021 16:08:00 +0100 Subject: [PATCH 319/330] Fix leak in test, and provide path to remove leak from library (#1169) * Fix leak in test, and provide path to remove leak from library * make doc change --- README.md | 1 + include/benchmark/benchmark.h | 3 +++ src/benchmark.cc | 4 ++++ test/benchmark_gtest.cc | 2 ++ 4 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 41cada9553..7947229499 100644 --- a/README.md +++ b/README.md @@ -1243,6 +1243,7 @@ int main(int argc, char** argv) { benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); } ``` diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 5b53debd25..290c5c5011 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -42,6 +42,7 @@ BENCHMARK(BM_StringCopy); int main(int argc, char** argv) { benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); return 0; } @@ -274,6 +275,7 @@ class BenchmarkReporter; class MemoryManager; void Initialize(int* argc, char** argv); +void Shutdown(); // Report to stdout all arguments in 'argv' as unrecognized except the first. // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). @@ -1314,6 +1316,7 @@ class Fixture : public internal::Benchmark { ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ ::benchmark::RunSpecifiedBenchmarks(); \ + ::benchmark::Shutdown(); \ return 0; \ } \ int main(int, char**) diff --git a/src/benchmark.cc b/src/benchmark.cc index 443ffbf9fa..c4392bfdcf 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -548,6 +548,10 @@ void Initialize(int* argc, char** argv) { internal::LogLevel() = FLAGS_v; } +void Shutdown() { + delete internal::global_context; +} + bool ReportUnrecognizedArguments(int argc, char** argv) { for (int i = 1; i < argc; ++i) { fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc index cbbf48b3b4..14a885ba46 100644 --- a/test/benchmark_gtest.cc +++ b/test/benchmark_gtest.cc @@ -143,6 +143,7 @@ TEST(AddCustomContext, Simple) { testing::UnorderedElementsAre(testing::Pair("foo", "bar"), testing::Pair("baz", "qux"))); + delete global_context; global_context = nullptr; } @@ -155,6 +156,7 @@ TEST(AddCustomContext, DuplicateKey) { EXPECT_THAT(*global_context, testing::UnorderedElementsAre(testing::Pair("foo", "bar"))); + delete global_context; global_context = nullptr; } From fbc31405b2a2838560e155d7aec937e135ae0d81 Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Thu, 3 Jun 2021 21:16:54 +0300 Subject: [PATCH 320/330] Random interleaving of benchmark repetitions - the sequel (fixes #1051) (#1163) Inspired by the original implementation by Hai Huang @haih-g from https://github.com/google/benchmark/pull/1105. The original implementation had design deficiencies that weren't really addressable without redesign, so it was reverted. In essence, the original implementation consisted of two separateable parts: * reducing the amount time each repetition is run for, and symmetrically increasing repetition count * running the repetitions in random order While it worked fine for the usual case, it broke down when user would specify repetitions (it would completely ignore that request), or specified per-repetition min time (while it would still adjust the repetition count, it would not adjust the per-repetition time, leading to much greater run times) Here, like i was originally suggesting in the original review, i'm separating the features, and only dealing with a single one - running repetitions in random order. Now that the runs/repetitions are no longer in-order, the tooling may wish to sort the output, and indeed `compare.py` has been updated to do that: #1168. --- README.md | 2 + docs/random_interleaving.md | 13 ++ include/benchmark/benchmark.h | 13 ++ src/benchmark.cc | 70 +++++++++-- src/benchmark_api_internal.h | 2 - src/benchmark_register.cc | 1 - src/benchmark_runner.cc | 48 ++++---- src/benchmark_runner.h | 26 ++-- test/CMakeLists.txt | 1 + test/benchmark_random_interleaving_gtest.cc | 126 ++++++++++++++++++++ 10 files changed, 254 insertions(+), 48 deletions(-) create mode 100644 docs/random_interleaving.md create mode 100644 test/benchmark_random_interleaving_gtest.cc diff --git a/README.md b/README.md index 7947229499..aa61cef1b1 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,8 @@ too (`-lkstat`). [Setting the Time Unit](#setting-the-time-unit) +[Random Interleaving](docs/random_interleaving.md) + [User-Requested Performance Counters](docs/perf_counters.md) [Preventing Optimization](#preventing-optimization) diff --git a/docs/random_interleaving.md b/docs/random_interleaving.md new file mode 100644 index 0000000000..c083036841 --- /dev/null +++ b/docs/random_interleaving.md @@ -0,0 +1,13 @@ + + +# Random Interleaving + +[Random Interleaving](https://github.com/google/benchmark/issues/1051) is a +technique to lower run-to-run variance. It randomly interleaves repetitions of a +microbenchmark with repetitions from other microbenchmarks in the same benchmark +test. Data shows it is able to lower run-to-run variance by +[40%](https://github.com/google/benchmark/issues/1051) on average. + +To use, you mainly need to set `--benchmark_enable_random_interleaving=true`, +and optionally specify non-zero repetition count `--benchmark_repetitions=9` +and optionally decrease the per-repetition time `--benchmark_min_time=0.1`. diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 290c5c5011..9b5480244d 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1472,6 +1472,19 @@ class BenchmarkReporter { int64_t max_bytes_used; }; + struct PerFamilyRunReports { + PerFamilyRunReports() : num_runs_total(0), num_runs_done(0) {} + + // How many runs will all instances of this benchmark perform? + int num_runs_total; + + // How many runs have happened already? + int num_runs_done; + + // The reports about (non-errneous!) runs of this family. + std::vector Runs; + }; + // Construct a BenchmarkReporter with the output stream set to 'std::cout' // and the error stream set to 'std::cerr' BenchmarkReporter(); diff --git a/src/benchmark.cc b/src/benchmark.cc index c4392bfdcf..89f64967bf 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -33,8 +33,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -73,6 +75,10 @@ DEFINE_double(benchmark_min_time, 0.5); // standard deviation of the runs will be reported. DEFINE_int32(benchmark_repetitions, 1); +// If set, enable random interleaving of repetitions of all benchmarks. +// See http://github.com/google/benchmark/issues/1051 for details. +DEFINE_bool(benchmark_enable_random_interleaving, false); + // Report the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are reported for // repeated benchmarks. Affects all reporters. @@ -297,23 +303,68 @@ void RunBenchmarks(const std::vector& benchmarks, context.name_field_width = name_field_width; // Keep track of running times of all instances of each benchmark family. - std::map> - complexity_reports; + std::map + per_family_reports; if (display_reporter->ReportContext(context) && (!file_reporter || file_reporter->ReportContext(context))) { FlushStreams(display_reporter); FlushStreams(file_reporter); + size_t num_repetitions_total = 0; + + std::vector runners; + runners.reserve(benchmarks.size()); for (const BenchmarkInstance& benchmark : benchmarks) { - std::vector* complexity_reports_for_family = - nullptr; + BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; if (benchmark.complexity() != oNone) - complexity_reports_for_family = - &complexity_reports[benchmark.family_index()]; + reports_for_family = &per_family_reports[benchmark.family_index()]; + + runners.emplace_back(benchmark, reports_for_family); + int num_repeats_of_this_instance = runners.back().GetNumRepeats(); + num_repetitions_total += num_repeats_of_this_instance; + if (reports_for_family) + reports_for_family->num_runs_total += num_repeats_of_this_instance; + } + assert(runners.size() == benchmarks.size() && "Unexpected runner count."); + + std::vector repetition_indices; + repetition_indices.reserve(num_repetitions_total); + for (size_t runner_index = 0, num_runners = runners.size(); + runner_index != num_runners; ++runner_index) { + const internal::BenchmarkRunner& runner = runners[runner_index]; + std::fill_n(std::back_inserter(repetition_indices), + runner.GetNumRepeats(), runner_index); + } + assert(repetition_indices.size() == num_repetitions_total && + "Unexpected number of repetition indexes."); + + if (FLAGS_benchmark_enable_random_interleaving) { + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(repetition_indices.begin(), repetition_indices.end(), g); + } - RunResults run_results = - RunBenchmark(benchmark, complexity_reports_for_family); + for (size_t repetition_index : repetition_indices) { + internal::BenchmarkRunner& runner = runners[repetition_index]; + runner.DoOneRepetition(); + if (runner.HasRepeatsRemaining()) continue; + // FIXME: report each repetition separately, not all of them in bulk. + + RunResults run_results = runner.GetResults(); + + // Maybe calculate complexity report + if (const auto* reports_for_family = runner.GetReportsForFamily()) { + if (reports_for_family->num_runs_done == + reports_for_family->num_runs_total) { + auto additional_run_stats = ComputeBigO(reports_for_family->Runs); + run_results.aggregates_only.insert(run_results.aggregates_only.end(), + additional_run_stats.begin(), + additional_run_stats.end()); + per_family_reports.erase( + (int)reports_for_family->Runs.front().family_index); + } + } Report(display_reporter, file_reporter, run_results); } @@ -471,6 +522,7 @@ void PrintUsageAndExit() { " [--benchmark_filter=]\n" " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n" " [--benchmark_format=]\n" @@ -495,6 +547,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_min_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", + &FLAGS_benchmark_enable_random_interleaving) || ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", &FLAGS_benchmark_report_aggregates_only) || ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index e2afbd833c..9296b7d2c8 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -39,8 +39,6 @@ class BenchmarkInstance { IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } - bool last_benchmark_instance; - State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 435c48b4aa..574462220e 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -166,7 +166,6 @@ bool BenchmarkFamilies::FindBenchmarks( const auto full_name = instance.name().str(); if ((re.Match(full_name) && !isNegativeFilter) || (!re.Match(full_name) && isNegativeFilter)) { - instance.last_benchmark_instance = (&args == &family->args_.back()); benchmarks->push_back(std::move(instance)); ++per_family_instance_index; diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 55d6cf15d2..6742d42dbe 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -143,9 +143,9 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, - std::vector* complexity_reports_) + BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), - complexity_reports(complexity_reports_), + reports_for_family(reports_for_family_), min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), @@ -172,22 +172,6 @@ BenchmarkRunner::BenchmarkRunner( perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; } - - for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { - DoOneRepetition(repetition_num); - } - - // Calculate additional statistics - run_results.aggregates_only = ComputeStats(run_results.non_aggregates); - - // Maybe calculate complexity report - if (complexity_reports && b.last_benchmark_instance) { - auto additional_run_stats = ComputeBigO(*complexity_reports); - run_results.aggregates_only.insert(run_results.aggregates_only.end(), - additional_run_stats.begin(), - additional_run_stats.end()); - complexity_reports->clear(); - } } BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { @@ -283,8 +267,10 @@ bool BenchmarkRunner::ShouldReportIterationResults( ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); } -void BenchmarkRunner::DoOneRepetition(int64_t repetition_index) { - const bool is_the_first_repetition = repetition_index == 0; +void BenchmarkRunner::DoOneRepetition() { + assert(HasRepeatsRemaining() && "Already done all repetitions?"); + + const bool is_the_first_repetition = num_repetitions_done == 0; IterationResults i; // We *may* be gradually increasing the length (iteration count) @@ -337,19 +323,25 @@ void BenchmarkRunner::DoOneRepetition(int64_t repetition_index) { // Ok, now actualy report. BenchmarkReporter::Run report = CreateRunReport(b, i.results, memory_iterations, memory_result, i.seconds, - repetition_index, repeats); + num_repetitions_done, repeats); - if (complexity_reports && !report.error_occurred) - complexity_reports->push_back(report); + if (reports_for_family) { + ++reports_for_family->num_runs_done; + if (!report.error_occurred) reports_for_family->Runs.push_back(report); + } run_results.non_aggregates.push_back(report); + + ++num_repetitions_done; } -RunResults RunBenchmark( - const benchmark::internal::BenchmarkInstance& b, - std::vector* complexity_reports) { - internal::BenchmarkRunner r(b, complexity_reports); - return r.get_results(); +RunResults&& BenchmarkRunner::GetResults() { + assert(!HasRepeatsRemaining() && "Did not run all repetitions yet?"); + + // Calculate additional statistics over the repetitions of this instance. + run_results.aggregates_only = ComputeStats(run_results.non_aggregates); + + return std::move(run_results); } } // end namespace internal diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 9730ad386c..8a855236b2 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -50,20 +50,34 @@ struct RunResults { class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, - std::vector* complexity_reports_); + BenchmarkReporter::PerFamilyRunReports* reports_for_family); - RunResults&& get_results() { return std::move(run_results); } + int GetNumRepeats() const { return repeats; } + + bool HasRepeatsRemaining() const { + return GetNumRepeats() != num_repetitions_done; + } + + void DoOneRepetition(); + + RunResults&& GetResults(); + + BenchmarkReporter::PerFamilyRunReports* GetReportsForFamily() const { + return reports_for_family; + }; private: RunResults run_results; const benchmark::internal::BenchmarkInstance& b; - std::vector* complexity_reports; + BenchmarkReporter::PerFamilyRunReports* reports_for_family; const double min_time; const int repeats; const bool has_explicit_iteration_count; + int num_repetitions_done = 0; + std::vector pool; IterationCount iters; // preserved between repetitions! @@ -83,14 +97,8 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const; bool ShouldReportIterationResults(const IterationResults& i) const; - - void DoOneRepetition(int64_t repetition_index); }; -RunResults RunBenchmark( - const benchmark::internal::BenchmarkInstance& b, - std::vector* complexity_reports); - } // namespace internal } // end namespace benchmark diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 012f5a8bcd..79cdf53b40 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -199,6 +199,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(benchmark_gtest) add_gtest(benchmark_name_gtest) + add_gtest(benchmark_random_interleaving_gtest) add_gtest(commandlineflags_gtest) add_gtest(statistics_gtest) add_gtest(string_util_gtest) diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc new file mode 100644 index 0000000000..8e28dab3f4 --- /dev/null +++ b/test/benchmark_random_interleaving_gtest.cc @@ -0,0 +1,126 @@ +#include +#include +#include + +#include "../src/commandlineflags.h" +#include "../src/string_util.h" +#include "benchmark/benchmark.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +DECLARE_bool(benchmark_enable_random_interleaving); +DECLARE_string(benchmark_filter); +DECLARE_int32(benchmark_repetitions); + +namespace benchmark { +namespace internal { +namespace { + +class EventQueue : public std::queue { + public: + void Put(const std::string& event) { push(event); } + + void Clear() { + while (!empty()) { + pop(); + } + } + + std::string Get() { + std::string event = front(); + pop(); + return event; + } +}; + +static EventQueue* queue = new EventQueue; + +class NullReporter : public BenchmarkReporter { + public: + bool ReportContext(const Context& /*context*/) override { return true; } + void ReportRuns(const std::vector& /* report */) override {} +}; + +class BenchmarkTest : public testing::Test { + public: + static void SetupHook(int /* num_threads */) { queue->push("Setup"); } + + static void TeardownHook(int /* num_threads */) { queue->push("Teardown"); } + + void Execute(const std::string& pattern) { + queue->Clear(); + + BenchmarkReporter* reporter = new NullReporter; + FLAGS_benchmark_filter = pattern; + RunSpecifiedBenchmarks(reporter); + delete reporter; + + queue->Put("DONE"); // End marker + } +}; + +static void BM_Match1(benchmark::State& state) { + const int64_t arg = state.range(0); + + for (auto _ : state) { + } + queue->Put(StrFormat("BM_Match1/%d", static_cast(arg))); +} +BENCHMARK(BM_Match1) + ->Iterations(100) + ->Arg(1) + ->Arg(2) + ->Arg(3) + ->Range(10, 80) + ->Args({90}) + ->Args({100}); + +TEST_F(BenchmarkTest, Match1) { + Execute("BM_Match1"); + ASSERT_EQ("BM_Match1/1", queue->Get()); + ASSERT_EQ("BM_Match1/2", queue->Get()); + ASSERT_EQ("BM_Match1/3", queue->Get()); + ASSERT_EQ("BM_Match1/10", queue->Get()); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("BM_Match1/90", queue->Get()); + ASSERT_EQ("BM_Match1/100", queue->Get()); + ASSERT_EQ("DONE", queue->Get()); +} + +TEST_F(BenchmarkTest, Match1WithRepetition) { + FLAGS_benchmark_repetitions = 2; + + Execute("BM_Match1/(64|80)"); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/64", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("BM_Match1/80", queue->Get()); + ASSERT_EQ("DONE", queue->Get()); +} + +TEST_F(BenchmarkTest, Match1WithRandomInterleaving) { + FLAGS_benchmark_enable_random_interleaving = true; + FLAGS_benchmark_repetitions = 100; + + std::map element_count; + std::map interleaving_count; + Execute("BM_Match1/(64|80)"); + for (int i = 0; i < 100; ++i) { + std::vector interleaving; + interleaving.push_back(queue->Get()); + interleaving.push_back(queue->Get()); + element_count[interleaving[0].c_str()]++; + element_count[interleaving[1].c_str()]++; + interleaving_count[StrFormat("%s,%s", interleaving[0].c_str(), + interleaving[1].c_str())]++; + } + EXPECT_EQ(element_count["BM_Match1/64"], 100) << "Unexpected repetitions."; + EXPECT_EQ(element_count["BM_Match1/80"], 100) << "Unexpected repetitions."; + EXPECT_GE(interleaving_count.size(), 2) << "Interleaving was not randomized."; + ASSERT_EQ("DONE", queue->Get()); +} + +} // namespace +} // namespace internal +} // namespace benchmark From bdd6c44787d42fe52107f834665afd1c3be0d547 Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 3 Jun 2021 19:45:02 +0100 Subject: [PATCH 321/330] Enable various sanitizer builds in github actions (#1167) * Enable various sanitizer builds in github actions * try with off the shelf versions * nope * specific version? * rats * oops * remove msan for now * reorder so env is set before building libc++ --- .github/workflows/sanitizer.yml | 83 +++++++++++++++++++++ .travis-libcxx-setup.sh => .libcxx-setup.sh | 2 +- .travis.yml | 2 +- 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/sanitizer.yml rename .travis-libcxx-setup.sh => .libcxx-setup.sh (96%) mode change 100644 => 100755 diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml new file mode 100644 index 0000000000..9543e16fa5 --- /dev/null +++ b/.github/workflows/sanitizer.yml @@ -0,0 +1,83 @@ +name: sanitizer + +on: + push: {} + pull_request: {} + +env: + CC: clang-3.8 + CXX: clang++-3.8 + EXTRA_CXX_FLAGS: "-stdlib=libc++" + UBSAN_OPTIONS: "print_stacktrace=1" + +jobs: + job: + name: ${{ matrix.sanitizer }}.${{ matrix.build_type }} + runs-on: ubuntu-16.04 + strategy: + fail-fast: false + matrix: + build_type: ['Debug', 'RelWithDebInfo'] + sanitizer: ['asan', 'ubsan', 'tsan'] + # TODO: add 'msan' above. currently failing and needs investigation. + steps: + - uses: actions/checkout@v2 + + - name: configure msan env + if: matrix.sanitizer == 'msan' + run: | + echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" >> $GITHUB_ENV + echo "LIBCXX_SANITIZER=MemoryWithOrigins" >> $GITHUB_ENV + + - name: configure ubsan env + if: matrix.sanitizer == 'ubsan' + run: | + echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all" >> $GITHUB_ENV + echo "LIBCXX_SANITIZER=Undefined" >> $GITHUB_ENV + + - name: configure asan env + if: matrix.sanitizer == 'asan' + run: | + echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=address -fno-sanitize-recover=all" >> $GITHUB_ENV + echo "LIBCXX_SANITIZER=Address" >> $GITHUB_ENV + + - name: configure tsan env + if: matrix.sanitizer == 'tsan' + run: | + echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" >> $GITHUB_ENV + echo "LIBCXX_SANITIZER=Thread" >> $GITHUB_ENV + + - name: install clang-3.8 + run: | + sudo apt update + sudo apt -y install clang-3.8 + + - name: install libc++ + run: "${GITHUB_WORKSPACE}/.libcxx-setup.sh" + + - name: create build environment + run: cmake -E make_directory ${{ runner.workspace }}/_build + + - name: configure cmake + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: > + cmake $GITHUB_WORKSPACE + -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF + -DBENCHMARK_ENABLE_LIBPFM=OFF + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON + -DCMAKE_C_COMPILER=${{ env.CC }} + -DCMAKE_CXX_COMPILER=${{ env.CXX }} + -DCMAKE_C_FLAGS="${{ env.EXTRA_FLAGS }}" + -DCMAKE_CXX_FLAGS="${{ env.EXTRA_FLAGS }} ${{ env.EXTRA_CXX_FLAGS }}" + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + + - name: build + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: cmake --build . --config ${{ matrix.build_type }} + + - name: test + shell: bash + working-directory: ${{ runner.workspace }}/_build + run: ctest -C ${{ matrix.build_type }} -VV diff --git a/.travis-libcxx-setup.sh b/.libcxx-setup.sh old mode 100644 new mode 100755 similarity index 96% rename from .travis-libcxx-setup.sh rename to .libcxx-setup.sh index a591743c6a..441460061d --- a/.travis-libcxx-setup.sh +++ b/.libcxx-setup.sh @@ -19,7 +19,7 @@ fi mkdir llvm-build && cd llvm-build cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} \ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr \ - -DLIBCXX_ABI_UNSTABLE=ON \ + -DLIBCXX_ABI_UNSTABLE=OFF \ -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ ../llvm-source diff --git a/.travis.yml b/.travis.yml index 4407af8d31..8cfed3d10d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -149,7 +149,7 @@ matrix: before_script: - if [ -n "${LIBCXX_BUILD}" ]; then - source .travis-libcxx-setup.sh; + source .libcxx-setup.sh; fi - if [ -n "${ENABLE_SANITIZER}" ]; then export EXTRA_OPTIONS="-DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF"; From 342409126b46dd7a33dc55f02d398bc43ba6b53b Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 4 Jun 2021 11:06:38 +0100 Subject: [PATCH 322/330] Use modern clang/libc++ for sanitizers (#1171) * Use modern clang/libc++ for sanitizers * update ubuntu * new llvm builds differently * clang, not clang-3.8 * just build what we need --- .github/.libcxx-setup.sh | 24 ++++++++++++++++++++++++ .github/workflows/sanitizer.yml | 19 +++++++------------ .libcxx-setup.sh | 28 ---------------------------- 3 files changed, 31 insertions(+), 40 deletions(-) create mode 100755 .github/.libcxx-setup.sh delete mode 100755 .libcxx-setup.sh diff --git a/.github/.libcxx-setup.sh b/.github/.libcxx-setup.sh new file mode 100755 index 0000000000..56008403ae --- /dev/null +++ b/.github/.libcxx-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Checkout LLVM sources +git clone --depth=1 https://github.com/llvm/llvm-project.git llvm-project + +# Setup libc++ options +if [ -z "$BUILD_32_BITS" ]; then + export BUILD_32_BITS=OFF && echo disabling 32 bit build +fi + +# Build and install libc++ (Use unstable ABI for better sanitizer coverage) +cd ./llvm-project +cmake -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${COMPILER} \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DLIBCXX_ABI_UNSTABLE=OFF \ + -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ + -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ + -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' \ + -S llvm -B llvm-build -G "Unix Makefiles" +make -C llvm-build -j3 cxx cxxabi +sudo make -C llvm-build install-cxx install-cxxabi +cd .. diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml index 9543e16fa5..fbc984492d 100644 --- a/.github/workflows/sanitizer.yml +++ b/.github/workflows/sanitizer.yml @@ -5,15 +5,15 @@ on: pull_request: {} env: - CC: clang-3.8 - CXX: clang++-3.8 + CC: clang + CXX: clang++ EXTRA_CXX_FLAGS: "-stdlib=libc++" UBSAN_OPTIONS: "print_stacktrace=1" jobs: job: name: ${{ matrix.sanitizer }}.${{ matrix.build_type }} - runs-on: ubuntu-16.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -28,7 +28,7 @@ jobs: run: | echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" >> $GITHUB_ENV echo "LIBCXX_SANITIZER=MemoryWithOrigins" >> $GITHUB_ENV - + - name: configure ubsan env if: matrix.sanitizer == 'ubsan' run: | @@ -47,13 +47,8 @@ jobs: echo "EXTRA_FLAGS=-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" >> $GITHUB_ENV echo "LIBCXX_SANITIZER=Thread" >> $GITHUB_ENV - - name: install clang-3.8 - run: | - sudo apt update - sudo apt -y install clang-3.8 - - - name: install libc++ - run: "${GITHUB_WORKSPACE}/.libcxx-setup.sh" + - name: install llvm stuff + run: "${GITHUB_WORKSPACE}/.github/.libcxx-setup.sh" - name: create build environment run: cmake -E make_directory ${{ runner.workspace }}/_build @@ -76,7 +71,7 @@ jobs: shell: bash working-directory: ${{ runner.workspace }}/_build run: cmake --build . --config ${{ matrix.build_type }} - + - name: test shell: bash working-directory: ${{ runner.workspace }}/_build diff --git a/.libcxx-setup.sh b/.libcxx-setup.sh deleted file mode 100755 index 441460061d..0000000000 --- a/.libcxx-setup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Install a newer CMake version -curl -sSL https://cmake.org/files/v3.6/cmake-3.6.1-Linux-x86_64.sh -o install-cmake.sh -chmod +x install-cmake.sh -sudo ./install-cmake.sh --prefix=/usr/local --skip-license - -# Checkout LLVM sources -git clone --depth=1 https://github.com/llvm-mirror/llvm.git llvm-source -git clone --depth=1 https://github.com/llvm-mirror/libcxx.git llvm-source/projects/libcxx -git clone --depth=1 https://github.com/llvm-mirror/libcxxabi.git llvm-source/projects/libcxxabi - -# Setup libc++ options -if [ -z "$BUILD_32_BITS" ]; then - export BUILD_32_BITS=OFF && echo disabling 32 bit build -fi - -# Build and install libc++ (Use unstable ABI for better sanitizer coverage) -mkdir llvm-build && cd llvm-build -cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr \ - -DLIBCXX_ABI_UNSTABLE=OFF \ - -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ - -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ - ../llvm-source -make cxx -j2 -sudo make install-cxxabi install-cxx -cd ../ From f90215f1cc2c22d32d96e903ea031278681e4adb Mon Sep 17 00:00:00 2001 From: huajingyun <74996522+huajingyun01@users.noreply.github.com> Date: Tue, 8 Jun 2021 04:26:24 -0500 Subject: [PATCH 323/330] Add support for new architecture loongarch (#1173) --- src/cycleclock.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cycleclock.h b/src/cycleclock.h index 9bef594bed..f22ca9f7d2 100644 --- a/src/cycleclock.h +++ b/src/cycleclock.h @@ -173,6 +173,10 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__loongarch__) + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #elif defined(__s390__) // Covers both s390 and s390x. // Return the CPU clock. uint64_t tsc; From e991355c02b93fe17713efe04cbc2e278e00fdbd Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Wed, 9 Jun 2021 11:52:12 +0300 Subject: [PATCH 324/330] [NFCI] Drop warning to satisfy clang's -Wunused-but-set-variable diag (#1174) Fixes https://github.com/google/benchmark/issues/1172 --- src/complexity.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/complexity.cc b/src/complexity.cc index d74b146922..29f7c3b031 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -82,7 +82,6 @@ std::string GetBigOString(BigO complexity) { LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { - double sigma_gn = 0.0; double sigma_gn_squared = 0.0; double sigma_time = 0.0; double sigma_time_gn = 0.0; @@ -90,7 +89,6 @@ LeastSq MinimalLeastSq(const std::vector& n, // Calculate least square fitting parameter for (size_t i = 0; i < n.size(); ++i) { double gn_i = fitting_curve(n[i]); - sigma_gn += gn_i; sigma_gn_squared += gn_i * gn_i; sigma_time += time[i]; sigma_time_gn += time[i] * gn_i; From 5b7518482c92df72dfdaf257beaf6e7175fefd9a Mon Sep 17 00:00:00 2001 From: Michael Lippautz Date: Tue, 15 Jun 2021 14:28:55 +0200 Subject: [PATCH 325/330] benchmark_runner.h: Remove superfluous semi colon (#1178) Some downstream projects (e.g. V8) treat warnings as errors and cannot roll the latest changes. --- src/benchmark_runner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 8a855236b2..693bdae93e 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -64,7 +64,7 @@ class BenchmarkRunner { BenchmarkReporter::PerFamilyRunReports* GetReportsForFamily() const { return reports_for_family; - }; + } private: RunResults run_results; From c932169e76f8bcdbc36f3b1e910642d529f66268 Mon Sep 17 00:00:00 2001 From: PCMan Date: Wed, 16 Jun 2021 19:56:24 +0800 Subject: [PATCH 326/330] Provide helpers to create integer lists for the given ranges. (#1179) This can be used together with ArgsProduct() to allow multiple ranges with different multipliers and mixing dense and sparse ranges. Example: BENCHMARK(MyTest)->ArgsProduct({ CreateRange(0, 1024, /*multi=*/32), CreateRange(0, 100, /*multi=*/4), CreateDenseRange(0, 4, /*step=*/1) }); Co-authored-by: Jen-yee Hong --- README.md | 17 +++++++++++++++++ include/benchmark/benchmark.h | 15 +++++++++++++++ src/benchmark_register.cc | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/README.md b/README.md index aa61cef1b1..809964261b 100644 --- a/README.md +++ b/README.md @@ -616,6 +616,23 @@ BENCHMARK(BM_SetInsert) ->Args({8<<10, 80}); ``` +For the most common scenarios, helper methods for creating a list of +integers for a given sparse or dense range are provided. + +```c++ +BENCHMARK(BM_SetInsert) + ->ArgsProduct({ + benchmark::CreateRange(8, 128, /*multi=*/2), + benchmark::CreateDenseRange(1, 4, /*step=*/1) + }) +// would generate the same benchmark arguments as +BENCHMARK(BM_SetInsert) + ->ArgsProduct({ + {8, 16, 32, 64, 128}, + {1, 2, 3, 4} + }); +``` + For more complex patterns of inputs, passing a custom function to `Apply` allows programmatic specification of an arbitrary set of arguments on which to run the benchmark. The following example enumerates a dense range on one parameter, diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h index 9b5480244d..8bfd852094 100644 --- a/include/benchmark/benchmark.h +++ b/include/benchmark/benchmark.h @@ -1649,6 +1649,21 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { BENCHMARK_UNREACHABLE(); } +// Creates a list of integer values for the given range and multiplier. +// This can be used together with ArgsProduct() to allow multiple ranges +// with different multiplers. +// Example: +// ArgsProduct({ +// CreateRange(0, 1024, /*multi=*/32), +// CreateRange(0, 100, /*multi=*/4), +// CreateDenseRange(0, 4, /*step=*/1), +// }); +std::vector CreateRange(int64_t lo, int64_t hi, int multi); + +// Creates a list of integer values for the given range and step. +std::vector CreateDenseRange(int64_t start, int64_t limit, + int step); + } // namespace benchmark #endif // BENCHMARK_BENCHMARK_H_ diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 574462220e..4c60bc736e 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -458,4 +458,20 @@ void ClearRegisteredBenchmarks() { internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks(); } +std::vector CreateRange(int64_t lo, int64_t hi, int multi) { + std::vector args; + internal::AddRange(&args, lo, hi, multi); + return args; +} + +std::vector CreateDenseRange(int64_t start, int64_t limit, + int step) { + CHECK_LE(start, limit); + std::vector args; + for (int64_t arg = start; arg <= limit; arg += step) { + args.push_back(arg); + } + return args; +} + } // end namespace benchmark From 62937f91b5c763a8e119d0c20c67b87bde8eff1c Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Fri, 18 Jun 2021 17:31:47 +0100 Subject: [PATCH 327/330] Add missing trailing commas (#1182) * Add missing trailing commas Fixes #1181 * Better trailing commas --- src/json_reporter.cc | 64 +++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 26898456f8..f3ddfb982d 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "benchmark/benchmark.h" -#include "complexity.h" - #include #include #include @@ -25,6 +22,8 @@ #include #include +#include "benchmark/benchmark.h" +#include "complexity.h" #include "string_util.h" #include "timers.h" @@ -35,34 +34,53 @@ extern std::map* global_context; namespace { -std::string StrEscape(const std::string & s) { +std::string StrEscape(const std::string& s) { std::string tmp; tmp.reserve(s.size()); for (char c : s) { switch (c) { - case '\b': tmp += "\\b"; break; - case '\f': tmp += "\\f"; break; - case '\n': tmp += "\\n"; break; - case '\r': tmp += "\\r"; break; - case '\t': tmp += "\\t"; break; - case '\\': tmp += "\\\\"; break; - case '"' : tmp += "\\\""; break; - default : tmp += c; break; + case '\b': + tmp += "\\b"; + break; + case '\f': + tmp += "\\f"; + break; + case '\n': + tmp += "\\n"; + break; + case '\r': + tmp += "\\r"; + break; + case '\t': + tmp += "\\t"; + break; + case '\\': + tmp += "\\\\"; + break; + case '"': + tmp += "\\\""; + break; + default: + tmp += c; + break; } } return tmp; } std::string FormatKV(std::string const& key, std::string const& value) { - return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); + return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), + StrEscape(value).c_str()); } std::string FormatKV(std::string const& key, const char* value) { - return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str()); + return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), + StrEscape(value).c_str()); } std::string FormatKV(std::string const& key, bool value) { - return StrFormat("\"%s\": %s", StrEscape(key).c_str(), value ? "true" : "false"); + return StrFormat("\"%s\": %s", StrEscape(key).c_str(), + value ? "true" : "false"); } std::string FormatKV(std::string const& key, int64_t value) { @@ -126,7 +144,9 @@ bool JSONReporter::ReportContext(const Context& context) { RoundDouble(info.cycles_per_second / 1000000.0)) << ",\n"; if (CPUInfo::Scaling::UNKNOWN != info.scaling) { - out << indent << FormatKV("cpu_scaling_enabled", info.scaling == CPUInfo::Scaling::ENABLED ? true : false) + out << indent + << FormatKV("cpu_scaling_enabled", + info.scaling == CPUInfo::Scaling::ENABLED ? true : false) << ",\n"; } @@ -139,8 +159,8 @@ bool JSONReporter::ReportContext(const Context& context) { out << cache_indent << FormatKV("type", CI.type) << ",\n"; out << cache_indent << FormatKV("level", static_cast(CI.level)) << ",\n"; - out << cache_indent - << FormatKV("size", static_cast(CI.size)) << ",\n"; + out << cache_indent << FormatKV("size", static_cast(CI.size)) + << ",\n"; out << cache_indent << FormatKV("num_sharing", static_cast(CI.num_sharing)) << "\n"; @@ -162,13 +182,15 @@ bool JSONReporter::ReportContext(const Context& context) { #else const char build_type[] = "debug"; #endif - out << indent << FormatKV("library_build_type", build_type) << "\n"; + out << indent << FormatKV("library_build_type", build_type); if (internal::global_context != nullptr) { - for (const auto& kv: *internal::global_context) { - out << indent << FormatKV(kv.first, kv.second) << "\n"; + for (const auto& kv : *internal::global_context) { + out << ",\n"; + out << indent << FormatKV(kv.first, kv.second); } } + out << "\n"; // Close context block and open the list of benchmarks. out << inner_indent << "},\n"; From 5da566042943805be87d19bed05092d21410f77c Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 24 Jun 2021 16:50:19 +0100 Subject: [PATCH 328/330] Move flags inside the `benchmark` namespace (#1185) This avoids clashes with other libraries that might define the same flags. --- src/benchmark.cc | 13 +++++++------ src/benchmark_runner.h | 8 ++------ test/benchmark_random_interleaving_gtest.cc | 3 ++- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 89f64967bf..71e2c94242 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -56,6 +56,7 @@ #include "thread_manager.h" #include "thread_timer.h" +namespace benchmark { // Print a list of benchmarks. This option overrides all other options. DEFINE_bool(benchmark_list_tests, false); @@ -112,20 +113,19 @@ DEFINE_string(benchmark_color, "auto"); // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. DEFINE_bool(benchmark_counters_tabular, false); -// The level of verbose logging to output -DEFINE_int32(v, 0); - // List of additional perf counters to collect, in libpfm format. For more // information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html DEFINE_string(benchmark_perf_counters, ""); -namespace benchmark { -namespace internal { - // Extra context to include in the output formatted as comma-separated key-value // pairs. Kept internal as it's only used for parsing from env/command line. DEFINE_kvpairs(benchmark_context, {}); +// The level of verbose logging to output +DEFINE_int32(v, 0); + +namespace internal { + std::map* global_context = nullptr; // FIXME: wouldn't LTO mess this up? @@ -530,6 +530,7 @@ void PrintUsageAndExit() { " [--benchmark_out_format=]\n" " [--benchmark_color={auto|true|false}]\n" " [--benchmark_counters_tabular={true|false}]\n" + " [--benchmark_perf_counters=,...]\n" " [--benchmark_context==,...]\n" " [--v=]\n"); exit(0); diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 693bdae93e..98ca9216d2 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -23,18 +23,14 @@ #include "perf_counters.h" #include "thread_manager.h" -DECLARE_double(benchmark_min_time); +namespace benchmark { +DECLARE_double(benchmark_min_time); DECLARE_int32(benchmark_repetitions); - DECLARE_bool(benchmark_report_aggregates_only); - DECLARE_bool(benchmark_display_aggregates_only); - DECLARE_string(benchmark_perf_counters); -namespace benchmark { - namespace internal { extern MemoryManager* memory_manager; diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc index 8e28dab3f4..6e48283533 100644 --- a/test/benchmark_random_interleaving_gtest.cc +++ b/test/benchmark_random_interleaving_gtest.cc @@ -8,11 +8,12 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +namespace benchmark { + DECLARE_bool(benchmark_enable_random_interleaving); DECLARE_string(benchmark_filter); DECLARE_int32(benchmark_repetitions); -namespace benchmark { namespace internal { namespace { From 6a5bf081d34b719aeea5a2eb2c83a17c2f24514f Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 24 Jun 2021 18:21:59 +0100 Subject: [PATCH 329/330] prefix macros to avoid clashes (#1186) --- src/benchmark.cc | 49 ++++++++++---------- src/benchmark_register.cc | 50 ++++++++++----------- src/benchmark_register.h | 18 ++++---- src/benchmark_runner.cc | 6 +-- src/benchmark_runner.h | 10 ++--- src/check.h | 39 ++++++++-------- src/colorprint.cc | 4 +- src/commandlineflags.h | 20 ++++----- src/complexity.cc | 11 ++--- src/csv_reporter.cc | 3 +- src/mutex.h | 2 +- src/perf_counters.h | 2 +- src/re.h | 2 +- src/reporter.cc | 2 +- src/statistics.cc | 8 ++-- src/sysinfo.cc | 2 +- src/thread_timer.h | 8 ++-- src/timers.cc | 4 +- test/benchmark_random_interleaving_gtest.cc | 6 +-- test/output_test.h | 8 ++-- test/output_test_helper.cc | 36 +++++++-------- test/perf_counters_gtest.cc | 6 +-- test/register_benchmark_test.cc | 6 +-- test/skip_with_error_test.cc | 10 ++--- 24 files changed, 158 insertions(+), 154 deletions(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index 71e2c94242..f9007e77c2 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -58,71 +58,71 @@ namespace benchmark { // Print a list of benchmarks. This option overrides all other options. -DEFINE_bool(benchmark_list_tests, false); +BM_DEFINE_bool(benchmark_list_tests, false); // A regular expression that specifies the set of benchmarks to execute. If // this flag is empty, or if this flag is the string \"all\", all benchmarks // linked into the binary are run. -DEFINE_string(benchmark_filter, "."); +BM_DEFINE_string(benchmark_filter, "."); // Minimum number of seconds we should run benchmark before results are // considered significant. For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -DEFINE_double(benchmark_min_time, 0.5); +BM_DEFINE_double(benchmark_min_time, 0.5); // The number of runs of each benchmark. If greater than 1, the mean and // standard deviation of the runs will be reported. -DEFINE_int32(benchmark_repetitions, 1); +BM_DEFINE_int32(benchmark_repetitions, 1); // If set, enable random interleaving of repetitions of all benchmarks. // See http://github.com/google/benchmark/issues/1051 for details. -DEFINE_bool(benchmark_enable_random_interleaving, false); +BM_DEFINE_bool(benchmark_enable_random_interleaving, false); // Report the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are reported for // repeated benchmarks. Affects all reporters. -DEFINE_bool(benchmark_report_aggregates_only, false); +BM_DEFINE_bool(benchmark_report_aggregates_only, false); // Display the result of each benchmark repetitions. When 'true' is specified // only the mean, standard deviation, and other statistics are displayed for // repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects // the display reporter, but *NOT* file reporter, which will still contain // all the output. -DEFINE_bool(benchmark_display_aggregates_only, false); +BM_DEFINE_bool(benchmark_display_aggregates_only, false); // The format to use for console output. // Valid values are 'console', 'json', or 'csv'. -DEFINE_string(benchmark_format, "console"); +BM_DEFINE_string(benchmark_format, "console"); // The format to use for file output. // Valid values are 'console', 'json', or 'csv'. -DEFINE_string(benchmark_out_format, "json"); +BM_DEFINE_string(benchmark_out_format, "json"); // The file to write additional output to. -DEFINE_string(benchmark_out, ""); +BM_DEFINE_string(benchmark_out, ""); // Whether to use colors in the output. Valid values: // 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if // the output is being sent to a terminal and the TERM environment variable is // set to a terminal type that supports colors. -DEFINE_string(benchmark_color, "auto"); +BM_DEFINE_string(benchmark_color, "auto"); // Whether to use tabular format when printing user counters to the console. // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. -DEFINE_bool(benchmark_counters_tabular, false); +BM_DEFINE_bool(benchmark_counters_tabular, false); // List of additional perf counters to collect, in libpfm format. For more // information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html -DEFINE_string(benchmark_perf_counters, ""); +BM_DEFINE_string(benchmark_perf_counters, ""); // Extra context to include in the output formatted as comma-separated key-value // pairs. Kept internal as it's only used for parsing from env/command line. -DEFINE_kvpairs(benchmark_context, {}); +BM_DEFINE_kvpairs(benchmark_context, {}); // The level of verbose logging to output -DEFINE_int32(v, 0); +BM_DEFINE_int32(v, 0); namespace internal { @@ -151,8 +151,9 @@ State::State(IterationCount max_iters, const std::vector& ranges, timer_(timer), manager_(manager), perf_counters_measurement_(perf_counters_measurement) { - CHECK(max_iterations != 0) << "At least one iteration must be run"; - CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; + BM_CHECK(max_iterations != 0) << "At least one iteration must be run"; + BM_CHECK_LT(thread_index, threads) + << "thread_index must be less than threads"; // Note: The use of offsetof below is technically undefined until C++17 // because State is not a standard layout type. However, all compilers @@ -181,21 +182,21 @@ State::State(IterationCount max_iters, const std::vector& ranges, void State::PauseTiming() { // Add in time accumulated so far - CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !error_occurred_); timer_->StopTimer(); if (perf_counters_measurement_) { auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); for (const auto& name_and_measurement : measurements) { auto name = name_and_measurement.first; auto measurement = name_and_measurement.second; - CHECK_EQ(counters[name], 0.0); + BM_CHECK_EQ(counters[name], 0.0); counters[name] = Counter(measurement, Counter::kAvgIterations); } } } void State::ResumeTiming() { - CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !error_occurred_); timer_->StartTimer(); if (perf_counters_measurement_) { perf_counters_measurement_->Start(); @@ -203,7 +204,7 @@ void State::ResumeTiming() { } void State::SkipWithError(const char* msg) { - CHECK(msg); + BM_CHECK(msg); error_occurred_ = true; { MutexLock l(manager_->GetBenchmarkMutex()); @@ -226,7 +227,7 @@ void State::SetLabel(const char* label) { } void State::StartKeepRunning() { - CHECK(!started_ && !finished_); + BM_CHECK(!started_ && !finished_); started_ = true; total_iterations_ = error_occurred_ ? 0 : max_iterations; manager_->StartStopBarrier(); @@ -234,7 +235,7 @@ void State::StartKeepRunning() { } void State::FinishKeepRunning() { - CHECK(started_ && (!finished_ || error_occurred_)); + BM_CHECK(started_ && (!finished_ || error_occurred_)); if (!error_occurred_) { PauseTiming(); } @@ -282,7 +283,7 @@ void RunBenchmarks(const std::vector& benchmarks, BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter) { // Note the file_reporter can be null. - CHECK(display_reporter != nullptr); + BM_CHECK(display_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 4c60bc736e..0a90be3d3c 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -111,7 +111,7 @@ void BenchmarkFamilies::ClearBenchmarks() { bool BenchmarkFamilies::FindBenchmarks( std::string spec, std::vector* benchmarks, std::ostream* ErrStream) { - CHECK(ErrStream); + BM_CHECK(ErrStream); auto& Err = *ErrStream; // Make regular expression out of command-line flag std::string error_msg; @@ -225,7 +225,7 @@ Benchmark* Benchmark::Name(const std::string& name) { } Benchmark* Benchmark::Arg(int64_t x) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); args_.push_back({x}); return this; } @@ -236,7 +236,7 @@ Benchmark* Benchmark::Unit(TimeUnit unit) { } Benchmark* Benchmark::Range(int64_t start, int64_t limit) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); std::vector arglist; AddRange(&arglist, start, limit, range_multiplier_); @@ -248,7 +248,7 @@ Benchmark* Benchmark::Range(int64_t start, int64_t limit) { Benchmark* Benchmark::Ranges( const std::vector>& ranges) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); std::vector> arglists(ranges.size()); for (std::size_t i = 0; i < ranges.size(); i++) { AddRange(&arglists[i], ranges[i].first, ranges[i].second, @@ -262,7 +262,7 @@ Benchmark* Benchmark::Ranges( Benchmark* Benchmark::ArgsProduct( const std::vector>& arglists) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(arglists.size())); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(arglists.size())); std::vector indices(arglists.size()); const std::size_t total = std::accumulate( @@ -289,20 +289,20 @@ Benchmark* Benchmark::ArgsProduct( } Benchmark* Benchmark::ArgName(const std::string& name) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); arg_names_ = {name}; return this; } Benchmark* Benchmark::ArgNames(const std::vector& names) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(names.size())); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(names.size())); arg_names_ = names; return this; } Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); - CHECK_LE(start, limit); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + BM_CHECK_LE(start, limit); for (int64_t arg = start; arg <= limit; arg += step) { args_.push_back({arg}); } @@ -310,7 +310,7 @@ Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) { } Benchmark* Benchmark::Args(const std::vector& args) { - CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(args.size())); + BM_CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(args.size())); args_.push_back(args); return this; } @@ -321,27 +321,27 @@ Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) { } Benchmark* Benchmark::RangeMultiplier(int multiplier) { - CHECK(multiplier > 1); + BM_CHECK(multiplier > 1); range_multiplier_ = multiplier; return this; } Benchmark* Benchmark::MinTime(double t) { - CHECK(t > 0.0); - CHECK(iterations_ == 0); + BM_CHECK(t > 0.0); + BM_CHECK(iterations_ == 0); min_time_ = t; return this; } Benchmark* Benchmark::Iterations(IterationCount n) { - CHECK(n > 0); - CHECK(IsZero(min_time_)); + BM_CHECK(n > 0); + BM_CHECK(IsZero(min_time_)); iterations_ = n; return this; } Benchmark* Benchmark::Repetitions(int n) { - CHECK(n > 0); + BM_CHECK(n > 0); repetitions_ = n; return this; } @@ -374,14 +374,14 @@ Benchmark* Benchmark::MeasureProcessCPUTime() { } Benchmark* Benchmark::UseRealTime() { - CHECK(!use_manual_time_) + BM_CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; use_real_time_ = true; return this; } Benchmark* Benchmark::UseManualTime() { - CHECK(!use_real_time_) + BM_CHECK(!use_real_time_) << "Cannot set UseRealTime and UseManualTime simultaneously."; use_manual_time_ = true; return this; @@ -405,14 +405,14 @@ Benchmark* Benchmark::ComputeStatistics(std::string name, } Benchmark* Benchmark::Threads(int t) { - CHECK_GT(t, 0); + BM_CHECK_GT(t, 0); thread_counts_.push_back(t); return this; } Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) { - CHECK_GT(min_threads, 0); - CHECK_GE(max_threads, min_threads); + BM_CHECK_GT(min_threads, 0); + BM_CHECK_GE(max_threads, min_threads); AddRange(&thread_counts_, min_threads, max_threads, 2); return this; @@ -420,9 +420,9 @@ Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) { Benchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads, int stride) { - CHECK_GT(min_threads, 0); - CHECK_GE(max_threads, min_threads); - CHECK_GE(stride, 1); + BM_CHECK_GT(min_threads, 0); + BM_CHECK_GE(max_threads, min_threads); + BM_CHECK_GE(stride, 1); for (auto i = min_threads; i < max_threads; i += stride) { thread_counts_.push_back(i); @@ -466,7 +466,7 @@ std::vector CreateRange(int64_t lo, int64_t hi, int multi) { std::vector CreateDenseRange(int64_t start, int64_t limit, int step) { - CHECK_LE(start, limit); + BM_CHECK_LE(start, limit); std::vector args; for (int64_t arg = start; arg <= limit; arg += step) { args.push_back(arg); diff --git a/src/benchmark_register.h b/src/benchmark_register.h index 09496607f2..7033dbf622 100644 --- a/src/benchmark_register.h +++ b/src/benchmark_register.h @@ -14,9 +14,9 @@ namespace internal { template typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, int mult) { - CHECK_GE(lo, 0); - CHECK_GE(hi, lo); - CHECK_GE(mult, 2); + BM_CHECK_GE(lo, 0); + BM_CHECK_GE(hi, lo); + BM_CHECK_GE(mult, 2); const size_t start_offset = dst->size(); @@ -38,10 +38,10 @@ AddPowers(std::vector* dst, T lo, T hi, int mult) { template void AddNegatedPowers(std::vector* dst, T lo, T hi, int mult) { // We negate lo and hi so we require that they cannot be equal to 'min'. - CHECK_GT(lo, std::numeric_limits::min()); - CHECK_GT(hi, std::numeric_limits::min()); - CHECK_GE(hi, lo); - CHECK_LE(hi, 0); + BM_CHECK_GT(lo, std::numeric_limits::min()); + BM_CHECK_GT(hi, std::numeric_limits::min()); + BM_CHECK_GE(hi, lo); + BM_CHECK_LE(hi, 0); // Add positive powers, then negate and reverse. // Casts necessary since small integers get promoted @@ -60,8 +60,8 @@ void AddRange(std::vector* dst, T lo, T hi, int mult) { static_assert(std::is_integral::value && std::is_signed::value, "Args type must be a signed integer"); - CHECK_GE(hi, lo); - CHECK_GE(mult, 2); + BM_CHECK_GE(hi, lo); + BM_CHECK_GE(mult, 2); // Add "lo" dst->push_back(lo); diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 6742d42dbe..a98345feaa 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -124,7 +124,7 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, : internal::ThreadTimer::Create()); State st = b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); - CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) + BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { MutexLock l(manager->GetBenchmarkMutex()); @@ -168,8 +168,8 @@ BenchmarkRunner::BenchmarkRunner( internal::ARM_DisplayReportAggregatesOnly); run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); - CHECK(FLAGS_benchmark_perf_counters.empty() || - perf_counters_measurement.IsValid()) + BM_CHECK(FLAGS_benchmark_perf_counters.empty() || + perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; } } diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 98ca9216d2..8427ce6a26 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -25,11 +25,11 @@ namespace benchmark { -DECLARE_double(benchmark_min_time); -DECLARE_int32(benchmark_repetitions); -DECLARE_bool(benchmark_report_aggregates_only); -DECLARE_bool(benchmark_display_aggregates_only); -DECLARE_string(benchmark_perf_counters); +BM_DECLARE_double(benchmark_min_time); +BM_DECLARE_int32(benchmark_repetitions); +BM_DECLARE_bool(benchmark_report_aggregates_only); +BM_DECLARE_bool(benchmark_display_aggregates_only); +BM_DECLARE_string(benchmark_perf_counters); namespace internal { diff --git a/src/check.h b/src/check.h index f5f8253f80..0efd13ff4d 100644 --- a/src/check.h +++ b/src/check.h @@ -23,8 +23,9 @@ BENCHMARK_NORETURN inline void CallAbortHandler() { std::abort(); // fallback to enforce noreturn } -// CheckHandler is the class constructed by failing CHECK macros. CheckHandler -// will log information about the failures and abort when it is destructed. +// CheckHandler is the class constructed by failing BM_CHECK macros. +// CheckHandler will log information about the failures and abort when it is +// destructed. class CheckHandler { public: CheckHandler(const char* check, const char* file, const char* func, int line) @@ -51,32 +52,32 @@ class CheckHandler { } // end namespace internal } // end namespace benchmark -// The CHECK macro returns a std::ostream object that can have extra information -// written to it. +// The BM_CHECK macro returns a std::ostream object that can have extra +// information written to it. #ifndef NDEBUG -#define CHECK(b) \ +#define BM_CHECK(b) \ (b ? ::benchmark::internal::GetNullLogInstance() \ : ::benchmark::internal::CheckHandler(#b, __FILE__, __func__, __LINE__) \ .GetLog()) #else -#define CHECK(b) ::benchmark::internal::GetNullLogInstance() +#define BM_CHECK(b) ::benchmark::internal::GetNullLogInstance() #endif // clang-format off // preserve whitespacing between operators for alignment -#define CHECK_EQ(a, b) CHECK((a) == (b)) -#define CHECK_NE(a, b) CHECK((a) != (b)) -#define CHECK_GE(a, b) CHECK((a) >= (b)) -#define CHECK_LE(a, b) CHECK((a) <= (b)) -#define CHECK_GT(a, b) CHECK((a) > (b)) -#define CHECK_LT(a, b) CHECK((a) < (b)) - -#define CHECK_FLOAT_EQ(a, b, eps) CHECK(std::fabs((a) - (b)) < (eps)) -#define CHECK_FLOAT_NE(a, b, eps) CHECK(std::fabs((a) - (b)) >= (eps)) -#define CHECK_FLOAT_GE(a, b, eps) CHECK((a) - (b) > -(eps)) -#define CHECK_FLOAT_LE(a, b, eps) CHECK((b) - (a) > -(eps)) -#define CHECK_FLOAT_GT(a, b, eps) CHECK((a) - (b) > (eps)) -#define CHECK_FLOAT_LT(a, b, eps) CHECK((b) - (a) > (eps)) +#define BM_CHECK_EQ(a, b) BM_CHECK((a) == (b)) +#define BM_CHECK_NE(a, b) BM_CHECK((a) != (b)) +#define BM_CHECK_GE(a, b) BM_CHECK((a) >= (b)) +#define BM_CHECK_LE(a, b) BM_CHECK((a) <= (b)) +#define BM_CHECK_GT(a, b) BM_CHECK((a) > (b)) +#define BM_CHECK_LT(a, b) BM_CHECK((a) < (b)) + +#define BM_CHECK_FLOAT_EQ(a, b, eps) BM_CHECK(std::fabs((a) - (b)) < (eps)) +#define BM_CHECK_FLOAT_NE(a, b, eps) BM_CHECK(std::fabs((a) - (b)) >= (eps)) +#define BM_CHECK_FLOAT_GE(a, b, eps) BM_CHECK((a) - (b) > -(eps)) +#define BM_CHECK_FLOAT_LE(a, b, eps) BM_CHECK((b) - (a) > -(eps)) +#define BM_CHECK_FLOAT_GT(a, b, eps) BM_CHECK((a) - (b) > (eps)) +#define BM_CHECK_FLOAT_LT(a, b, eps) BM_CHECK((b) - (a) > (eps)) //clang-format on #endif // CHECK_H_ diff --git a/src/colorprint.cc b/src/colorprint.cc index fff6a98818..afaa55dd54 100644 --- a/src/colorprint.cc +++ b/src/colorprint.cc @@ -94,7 +94,7 @@ std::string FormatString(const char* msg, va_list args) { va_end(args_cp); // currently there is no error handling for failure, so this is hack. - CHECK(ret >= 0); + BM_CHECK(ret >= 0); if (ret == 0) // handle empty expansion return {}; @@ -105,7 +105,7 @@ std::string FormatString(const char* msg, va_list args) { size = (size_t)ret + 1; // + 1 for the null byte std::unique_ptr buff(new char[size]); ret = vsnprintf(buff.get(), size, msg, args); - CHECK(ret > 0 && ((size_t)ret) < size); + BM_CHECK(ret > 0 && ((size_t)ret) < size); return buff.get(); } } diff --git a/src/commandlineflags.h b/src/commandlineflags.h index 0c988cccb3..5baaf11784 100644 --- a/src/commandlineflags.h +++ b/src/commandlineflags.h @@ -9,23 +9,23 @@ #define FLAG(name) FLAGS_##name // Macros for declaring flags. -#define DECLARE_bool(name) extern bool FLAG(name) -#define DECLARE_int32(name) extern int32_t FLAG(name) -#define DECLARE_double(name) extern double FLAG(name) -#define DECLARE_string(name) extern std::string FLAG(name) -#define DECLARE_kvpairs(name) \ +#define BM_DECLARE_bool(name) extern bool FLAG(name) +#define BM_DECLARE_int32(name) extern int32_t FLAG(name) +#define BM_DECLARE_double(name) extern double FLAG(name) +#define BM_DECLARE_string(name) extern std::string FLAG(name) +#define BM_DECLARE_kvpairs(name) \ extern std::map FLAG(name) // Macros for defining flags. -#define DEFINE_bool(name, default_val) \ +#define BM_DEFINE_bool(name, default_val) \ bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) -#define DEFINE_int32(name, default_val) \ +#define BM_DEFINE_int32(name, default_val) \ int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) -#define DEFINE_double(name, default_val) \ +#define BM_DEFINE_double(name, default_val) \ double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) -#define DEFINE_string(name, default_val) \ +#define BM_DEFINE_string(name, default_val) \ std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) -#define DEFINE_kvpairs(name, default_val) \ +#define BM_DEFINE_kvpairs(name, default_val) \ std::map FLAG(name) = \ benchmark::KvPairsFromEnv(#name, default_val) diff --git a/src/complexity.cc b/src/complexity.cc index 29f7c3b031..7757315dc3 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -123,10 +123,10 @@ LeastSq MinimalLeastSq(const std::vector& n, // fitting curve. LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { - CHECK_EQ(n.size(), time.size()); - CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two - // benchmark runs are given - CHECK_NE(complexity, oNone); + BM_CHECK_EQ(n.size(), time.size()); + BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two + // benchmark runs are given + BM_CHECK_NE(complexity, oNone); LeastSq best_fit; @@ -167,7 +167,8 @@ std::vector ComputeBigO( // Populate the accumulators. for (const Run& run : reports) { - CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; + BM_CHECK_GT(run.complexity_n, 0) + << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); real_time.push_back(run.real_accumulated_time / run.iterations); cpu_time.push_back(run.cpu_accumulated_time / run.iterations); diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc index af2c18fc8a..9bd7121daf 100644 --- a/src/csv_reporter.cc +++ b/src/csv_reporter.cc @@ -85,7 +85,8 @@ void CSVReporter::ReportRuns(const std::vector& reports) { for (const auto& cnt : run.counters) { if (cnt.first == "bytes_per_second" || cnt.first == "items_per_second") continue; - CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) + BM_CHECK(user_counter_names_.find(cnt.first) != + user_counter_names_.end()) << "All counters must be present in each run. " << "Counter named \"" << cnt.first << "\" was not in a run after being added to the header"; diff --git a/src/mutex.h b/src/mutex.h index 9cc414ec46..bec78d9e5f 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -130,7 +130,7 @@ class Barrier { // entered the barrier. Returns iff this is the last thread to // enter the barrier. bool createBarrier(MutexLock& ml) REQUIRES(lock_) { - CHECK_LT(entered_, running_threads_); + BM_CHECK_LT(entered_, running_threads_); entered_++; if (entered_ < running_threads_) { // Wait for all threads to enter diff --git a/src/perf_counters.h b/src/perf_counters.h index b6629b9907..47ca1385e2 100644 --- a/src/perf_counters.h +++ b/src/perf_counters.h @@ -42,7 +42,7 @@ namespace internal { class PerfCounterValues { public: explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { - CHECK_LE(nr_counters_, kMaxCounters); + BM_CHECK_LE(nr_counters_, kMaxCounters); } uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } diff --git a/src/re.h b/src/re.h index fbe25037b4..630046782d 100644 --- a/src/re.h +++ b/src/re.h @@ -126,7 +126,7 @@ inline bool Regex::Init(const std::string& spec, std::string* error) { // regerror returns the number of bytes necessary to null terminate // the string, so we move that when assigning to error. - CHECK_NE(needed, 0); + BM_CHECK_NE(needed, 0); error->assign(errbuf, needed - 1); delete[] errbuf; diff --git a/src/reporter.cc b/src/reporter.cc index 14dd40dc72..c720a9df1d 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -38,7 +38,7 @@ BenchmarkReporter::~BenchmarkReporter() {} void BenchmarkReporter::PrintBasicContext(std::ostream *out, Context const &context) { - CHECK(out) << "cannot be null"; + BM_CHECK(out) << "cannot be null"; auto &Out = *out; Out << LocalDateTimeString() << "\n"; diff --git a/src/statistics.cc b/src/statistics.cc index 57472b9ff9..5f82c27372 100644 --- a/src/statistics.cc +++ b/src/statistics.cc @@ -112,22 +112,22 @@ std::vector ComputeStats( it = counter_stats.find(cnt.first); it->second.s.reserve(reports.size()); } else { - CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); + BM_CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); } } } // Populate the accumulators. for (Run const& run : reports) { - CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); - CHECK_EQ(run_iterations, run.iterations); + BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); + BM_CHECK_EQ(run_iterations, run.iterations); if (run.error_occurred) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); // user counters for (auto const& cnt : run.counters) { auto it = counter_stats.find(cnt.first); - CHECK_NE(it, counter_stats.end()); + BM_CHECK_NE(it, counter_stats.end()); it->second.s.emplace_back(cnt.second); } } diff --git a/src/sysinfo.cc b/src/sysinfo.cc index c1969ea2d3..abe49ba716 100644 --- a/src/sysinfo.cc +++ b/src/sysinfo.cc @@ -135,7 +135,7 @@ struct ValueUnion { template std::array GetAsArray() { const int ArrSize = sizeof(T) * N; - CHECK_LE(ArrSize, Size); + BM_CHECK_LE(ArrSize, Size); std::array Arr; std::memcpy(Arr.data(), data(), ArrSize); return Arr; diff --git a/src/thread_timer.h b/src/thread_timer.h index 1703ca0d6f..eb23f59561 100644 --- a/src/thread_timer.h +++ b/src/thread_timer.h @@ -28,7 +28,7 @@ class ThreadTimer { // Called by each thread void StopTimer() { - CHECK(running_); + BM_CHECK(running_); running_ = false; real_time_used_ += ChronoClockNow() - start_real_time_; // Floating point error can result in the subtraction producing a negative @@ -44,19 +44,19 @@ class ThreadTimer { // REQUIRES: timer is not running double real_time_used() const { - CHECK(!running_); + BM_CHECK(!running_); return real_time_used_; } // REQUIRES: timer is not running double cpu_time_used() const { - CHECK(!running_); + BM_CHECK(!running_); return cpu_time_used_; } // REQUIRES: timer is not running double manual_time_used() const { - CHECK(!running_); + BM_CHECK(!running_); return manual_time_used_; } diff --git a/src/timers.cc b/src/timers.cc index af4767dff9..1f05574269 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -225,7 +225,7 @@ std::string LocalDateTimeString() { tz_len = ::snprintf(tz_offset, sizeof(tz_offset), "%c%02li:%02li", tz_offset_sign, offset_minutes / 100, offset_minutes % 100); - CHECK(tz_len == kTzOffsetLen); + BM_CHECK(tz_len == kTzOffsetLen); ((void)tz_len); // Prevent unused variable warning in optimized build. } else { // Unknown offset. RFC3339 specifies that unknown local offsets should be @@ -242,7 +242,7 @@ std::string LocalDateTimeString() { timestamp_len = std::strftime(storage, sizeof(storage), "%Y-%m-%dT%H:%M:%S", timeinfo_p); - CHECK(timestamp_len == kTimestampLen); + BM_CHECK(timestamp_len == kTimestampLen); // Prevent unused variable warning in optimized build. ((void)kTimestampLen); diff --git a/test/benchmark_random_interleaving_gtest.cc b/test/benchmark_random_interleaving_gtest.cc index 6e48283533..f68c680e94 100644 --- a/test/benchmark_random_interleaving_gtest.cc +++ b/test/benchmark_random_interleaving_gtest.cc @@ -10,9 +10,9 @@ namespace benchmark { -DECLARE_bool(benchmark_enable_random_interleaving); -DECLARE_string(benchmark_filter); -DECLARE_int32(benchmark_repetitions); +BM_DECLARE_bool(benchmark_enable_random_interleaving); +BM_DECLARE_string(benchmark_filter); +BM_DECLARE_int32(benchmark_repetitions); namespace internal { namespace { diff --git a/test/output_test.h b/test/output_test.h index 15368f9b68..3b12f3777d 100644 --- a/test/output_test.h +++ b/test/output_test.h @@ -143,12 +143,12 @@ struct Results { template T Results::GetAs(const char* entry_name) const { auto* sv = Get(entry_name); - CHECK(sv != nullptr && !sv->empty()); + BM_CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; ss << *sv; T out; ss >> out; - CHECK(!ss.fail()); + BM_CHECK(!ss.fail()); return out; } @@ -159,7 +159,7 @@ T Results::GetAs(const char* entry_name) const { // clang-format off #define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \ - CONCAT(CHECK_, relationship) \ + CONCAT(BM_CHECK_, relationship) \ (entry.getfn< var_type >(var_name), (value)) << "\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ << __FILE__ << ":" << __LINE__ << ": " \ @@ -170,7 +170,7 @@ T Results::GetAs(const char* entry_name) const { // check with tolerance. eps_factor is the tolerance window, which is // interpreted relative to value (eg, 0.1 means 10% of value). #define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ - CONCAT(CHECK_FLOAT_, relationship) \ + CONCAT(BM_CHECK_FLOAT_, relationship) \ (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ << __FILE__ << ":" << __LINE__ << ": " \ diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index b8ef120574..d3ec34dbad 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -94,27 +94,27 @@ void CheckCase(std::stringstream& remaining_output, TestCase const& TC, bool on_first = true; std::string line; while (remaining_output.eof() == false) { - CHECK(remaining_output.good()); + BM_CHECK(remaining_output.good()); std::getline(remaining_output, line); if (on_first) { first_line = line; on_first = false; } for (const auto& NC : not_checks) { - CHECK(!NC.regex->Match(line)) + BM_CHECK(!NC.regex->Match(line)) << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" << NC.regex_str << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } if (TC.regex->Match(line)) return; - CHECK(TC.match_rule != MR_Next) + BM_CHECK(TC.match_rule != MR_Next) << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str << "\"" << "\n actual regex string \"" << TC.substituted_regex << "\"" << "\n started matching near: " << first_line; } - CHECK(remaining_output.eof() == false) + BM_CHECK(remaining_output.eof() == false) << "End of output reached before match for regex \"" << TC.regex_str << "\" was found" << "\n actual regex string \"" << TC.substituted_regex << "\"" @@ -144,7 +144,7 @@ class TestReporter : public benchmark::BenchmarkReporter { bool first = true; for (auto rep : reporters_) { bool new_ret = rep->ReportContext(context); - CHECK(first || new_ret == last_ret) + BM_CHECK(first || new_ret == last_ret) << "Reports return different values for ReportContext"; first = false; last_ret = new_ret; @@ -226,7 +226,7 @@ void ResultsChecker::CheckResults(std::stringstream& output) { std::string line; bool on_first = true; while (output.eof() == false) { - CHECK(output.good()); + BM_CHECK(output.good()); std::getline(output, line); if (on_first) { SetHeader_(line); // this is important @@ -261,9 +261,9 @@ void ResultsChecker::SetHeader_(const std::string& csv_header) { // set the values for a benchmark void ResultsChecker::SetValues_(const std::string& entry_csv_line) { if (entry_csv_line.empty()) return; // some lines are empty - CHECK(!field_names.empty()); + BM_CHECK(!field_names.empty()); auto vals = SplitCsv_(entry_csv_line); - CHECK_EQ(vals.size(), field_names.size()); + BM_CHECK_EQ(vals.size(), field_names.size()); results.emplace_back(vals[0]); // vals[0] is the benchmark name auto& entry = results.back(); for (size_t i = 1, e = vals.size(); i < e; ++i) { @@ -278,7 +278,7 @@ std::vector ResultsChecker::SplitCsv_(const std::string& line) { if (!field_names.empty()) out.reserve(field_names.size()); size_t prev = 0, pos = line.find_first_of(','), curr = pos; while (pos != line.npos) { - CHECK(curr > 0); + BM_CHECK(curr > 0); if (line[prev] == '"') ++prev; if (line[curr - 1] == '"') --curr; out.push_back(line.substr(prev, curr - prev)); @@ -309,7 +309,7 @@ int Results::NumThreads() const { ss << name.substr(pos + 9, end); int num = 1; ss >> num; - CHECK(!ss.fail()); + BM_CHECK(!ss.fail()); return num; } @@ -318,11 +318,11 @@ double Results::NumIterations() const { } double Results::GetTime(BenchmarkTime which) const { - CHECK(which == kCpuTime || which == kRealTime); + BM_CHECK(which == kCpuTime || which == kRealTime); const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; double val = GetAs(which_str); auto unit = Get("time_unit"); - CHECK(unit); + BM_CHECK(unit); if (*unit == "ns") { return val * 1.e-9; } else if (*unit == "us") { @@ -332,7 +332,7 @@ double Results::GetTime(BenchmarkTime which) const { } else if (*unit == "s") { return val; } else { - CHECK(1 == 0) << "unknown time unit: " << *unit; + BM_CHECK(1 == 0) << "unknown time unit: " << *unit; return 0; } } @@ -348,10 +348,10 @@ TestCase::TestCase(std::string re, int rule) regex(std::make_shared()) { std::string err_str; regex->Init(substituted_regex, &err_str); - CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex - << "\"" - << "\n originally \"" << regex_str << "\"" - << "\n got error: " << err_str; + BM_CHECK(err_str.empty()) + << "Could not construct regex \"" << substituted_regex << "\"" + << "\n originally \"" << regex_str << "\"" + << "\n got error: " << err_str; } int AddCases(TestCaseID ID, std::initializer_list il) { @@ -438,7 +438,7 @@ void RunOutputTests(int argc, char* argv[]) { // the checks to subscribees. auto& csv = TestCases[2]; // would use == but gcc spits a warning - CHECK(std::strcmp(csv.name, "CSVReporter") == 0); + BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); } diff --git a/test/perf_counters_gtest.cc b/test/perf_counters_gtest.cc index 2a2868a715..ddeddf4f4d 100644 --- a/test/perf_counters_gtest.cc +++ b/test/perf_counters_gtest.cc @@ -103,10 +103,10 @@ size_t do_work() { void measure(size_t threadcount, PerfCounterValues* values1, PerfCounterValues* values2) { - CHECK_NE(values1, nullptr); - CHECK_NE(values2, nullptr); + BM_CHECK_NE(values1, nullptr); + BM_CHECK_NE(values2, nullptr); std::vector threads(threadcount); - auto work = [&]() { CHECK(do_work() > 1000); }; + auto work = [&]() { BM_CHECK(do_work() > 1000); }; // We need to first set up the counters, then start the threads, so the // threads would inherit the counters. But later, we need to first destroy the diff --git a/test/register_benchmark_test.cc b/test/register_benchmark_test.cc index c027eabaca..8613acba4f 100644 --- a/test/register_benchmark_test.cc +++ b/test/register_benchmark_test.cc @@ -30,13 +30,13 @@ struct TestCase { void CheckRun(Run const& run) const { // clang-format off - CHECK(name == run.benchmark_name()) << "expected " << name << " got " + BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); if (label) { - CHECK(run.report_label == label) << "expected " << label << " got " + BM_CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; } else { - CHECK(run.report_label == ""); + BM_CHECK(run.report_label == ""); } // clang-format on } diff --git a/test/skip_with_error_test.cc b/test/skip_with_error_test.cc index 827966e9df..1156bc0ad6 100644 --- a/test/skip_with_error_test.cc +++ b/test/skip_with_error_test.cc @@ -33,14 +33,14 @@ struct TestCase { typedef benchmark::BenchmarkReporter::Run Run; void CheckRun(Run const& run) const { - CHECK(name == run.benchmark_name()) + BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - CHECK(error_occurred == run.error_occurred); - CHECK(error_message == run.error_message); + BM_CHECK(error_occurred == run.error_occurred); + BM_CHECK(error_message == run.error_message); if (error_occurred) { - // CHECK(run.iterations == 0); + // BM_CHECK(run.iterations == 0); } else { - CHECK(run.iterations != 0); + BM_CHECK(run.iterations != 0); } } }; From 42a34570bfaa7416d334b2eb7dcad6ba266ebaff Mon Sep 17 00:00:00 2001 From: Dominic Hamon Date: Thu, 24 Jun 2021 18:39:20 +0100 Subject: [PATCH 330/330] prefix VLOG --- src/benchmark_runner.cc | 8 ++++---- src/log.h | 2 +- test/output_test_helper.cc | 13 +++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index a98345feaa..485bbc6e85 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -175,7 +175,7 @@ BenchmarkRunner::BenchmarkRunner( } BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { - VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; + BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; std::unique_ptr manager; manager.reset(new internal::ThreadManager(b.threads())); @@ -210,8 +210,8 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { // If we were measuring whole-process CPU usage, adjust the CPU time too. if (b.measure_process_cpu_time()) i.results.cpu_time_used /= b.threads(); - VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" - << i.results.real_time_used << "\n"; + BM_VLOG(2) << "Ran in " << i.results.cpu_time_used << "/" + << i.results.real_time_used << "\n"; // By using KeepRunningBatch a benchmark can iterate more times than // requested, so take the iteration count from i.results. @@ -249,7 +249,7 @@ IterationCount BenchmarkRunner::PredictNumItersNeeded( // But we do have *some* sanity limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); - VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; + BM_VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; return next_iters; // round up before conversion to integer. } diff --git a/src/log.h b/src/log.h index 47d0c35c01..48c071aded 100644 --- a/src/log.h +++ b/src/log.h @@ -67,7 +67,7 @@ inline LogType& GetLogInstanceForLevel(int level) { } // end namespace benchmark // clang-format off -#define VLOG(x) \ +#define BM_VLOG(x) \ (::benchmark::internal::GetLogInstanceForLevel(x) << "-- LOG(" << x << "):" \ " ") // clang-format on diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc index d3ec34dbad..4603459987 100644 --- a/test/output_test_helper.cc +++ b/test/output_test_helper.cc @@ -10,6 +10,7 @@ #include "../src/benchmark_api_internal.h" #include "../src/check.h" // NOTE: check.h is for internal use only! +#include "../src/log.h" // NOTE: log.h is for internal use only #include "../src/re.h" // NOTE: re.h is for internal use only #include "output_test.h" @@ -237,18 +238,18 @@ void ResultsChecker::CheckResults(std::stringstream& output) { } // finally we can call the subscribed check functions for (const auto& p : check_patterns) { - VLOG(2) << "--------------------------------\n"; - VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; + BM_VLOG(2) << "--------------------------------\n"; + BM_VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; for (const auto& r : results) { if (!p.regex->Match(r.name)) { - VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; + BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; } else { - VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; + BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } - VLOG(1) << "Checking results of " << r.name << ": ... \n"; + BM_VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); - VLOG(1) << "Checking results of " << r.name << ": OK.\n"; + BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n"; } } }