Remove the separately compiled test for Benchmarking Macros

Thanks to the changes to compilation model, the tests for benchmarking
macros have been made part of the normal test run. This means that
the only purpose these separately compiled tests served was to waste
CI time.

tests/CMakeLists.txt

@@ -123,6 +123,7 @@ endif()
 add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 set_tests_properties(RunTests PROPERTIES
     FAIL_REGULAR_EXPRESSION "Filters:"
+    COST 60
 )
 
 # Because CTest does not allow us to check both return code _and_ expected

tests/ExtraTests/CMakeLists.txt

@@ -119,18 +119,6 @@ set_tests_properties(
 )
 
 
-add_executable(BenchmarkingMacros ${TESTS_DIR}/X20-BenchmarkingMacros.cpp)
-target_link_libraries( BenchmarkingMacros Catch2_buildall_interface )
-
-add_test(NAME BenchmarkingMacros COMMAND BenchmarkingMacros -r console -s)
-set_tests_properties(
-    BenchmarkingMacros
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "benchmark name                                  samples       iterations    estimated"
-    COST 80 # We know that this is either the most, or second most,
-   # expensive test in the test suite, so we give it high estimate for CI runs
-)
-
 # This test touches windows.h, so it should only be compiled under msvc
 if (MSVC)
     # This test fails if it does not compile and succeeds otherwise
@@ -142,13 +130,14 @@ if (MSVC)
     add_test(NAME WindowsHeader COMMAND WindowsHeader -r compact)
 endif()
 
-add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp)
-add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break)
-set_tests_properties(
-    DebugBreakMacros
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "Pretty please, break into debugger"
-)
+#add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp)
+#target_link_libraries(DebugBreakMacros Catch2)
+#add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break)
+#set_tests_properties(
+#    DebugBreakMacros
+#  PROPERTIES
+#    PASS_REGULAR_EXPRESSION "Pretty please, break into debugger"
+#)
 
 set( EXTRA_TEST_BINARIES
     PrefixedMacros
@@ -157,8 +146,7 @@ set( EXTRA_TEST_BINARIES
     DisabledExceptions-CustomHandler
     FallbackStringifier
     DisableStringification
-    BenchmarkingMacros
-    DebugBreakMacros
+#    DebugBreakMacros
 )
 
 # Shared config

tests/ExtraTests/X20-BenchmarkingMacros.cpp

@@ -1,127 +0,0 @@
-// X20-BenchmarkingMacros.cpp
-// Test that the benchmarking support macros compile properly with the single header
-
-#include <catch2/catch_default_main.hpp>
-#include <catch2/catch_test_macros.hpp>
-#include <catch2/benchmark/catch_benchmark.hpp>
-#include <catch2/catch_generators_specific.hpp>
-
-namespace {
-std::uint64_t factorial(std::uint64_t number) {
-    if (number < 2) {
-        return 1;
-    }
-    return number * factorial(number - 1);
-}
-}
-
-TEST_CASE("Benchmark factorial", "[benchmark]") {
-    CHECK(factorial(0) == 1);
-    // some more asserts..
-    CHECK(factorial(10) == 3628800);
-
-    BENCHMARK("factorial 10") {
-        return factorial(10);
-    };
-
-    CHECK(factorial(14) == 87178291200ull);
-    BENCHMARK("factorial 14") {
-        return factorial(14);
-    };
-}
-
-TEST_CASE("Benchmark containers", "[.][benchmark]") {
-    static const int size = 100;
-
-    std::vector<int> v;
-    std::map<int, int> m;
-
-    SECTION("without generator") {
-        BENCHMARK("Load up a vector") {
-            v = std::vector<int>();
-            for (int i = 0; i < size; ++i)
-                v.push_back(i);
-        };
-        REQUIRE(v.size() == size);
-
-        // test optimizer control
-        BENCHMARK("Add up a vector's content") {
-            uint64_t add = 0;
-            for (int i = 0; i < size; ++i)
-                add += v[i];
-            return add;
-        };
-
-        BENCHMARK("Load up a map") {
-            m = std::map<int, int>();
-            for (int i = 0; i < size; ++i)
-                m.insert({ i, i + 1 });
-        };
-        REQUIRE(m.size() == size);
-
-        BENCHMARK("Reserved vector") {
-            v = std::vector<int>();
-            v.reserve(size);
-            for (int i = 0; i < size; ++i)
-                v.push_back(i);
-        };
-        REQUIRE(v.size() == size);
-
-        BENCHMARK("Resized vector") {
-            v = std::vector<int>();
-            v.resize(size);
-            for (int i = 0; i < size; ++i)
-                v[i] = i;
-        };
-        REQUIRE(v.size() == size);
-
-        int array[size];
-        BENCHMARK("A fixed size array that should require no allocations") {
-            for (int i = 0; i < size; ++i)
-                array[i] = i;
-        };
-        int sum = 0;
-        for (int i = 0; i < size; ++i)
-            sum += array[i];
-        REQUIRE(sum > size);
-
-        SECTION("XYZ") {
-
-            BENCHMARK_ADVANCED("Load up vector with chronometer")(Catch::Benchmark::Chronometer meter) {
-                std::vector<int> k;
-                meter.measure([&](int idx) {
-                    k = std::vector<int>();
-                    for (int i = 0; i < size; ++i)
-                        k.push_back(idx);
-                });
-                REQUIRE(k.size() == size);
-            };
-
-            int runs = 0;
-            BENCHMARK("Fill vector indexed", benchmarkIndex) {
-                v = std::vector<int>();
-                v.resize(size);
-                for (int i = 0; i < size; ++i)
-                    v[i] = benchmarkIndex;
-                runs = benchmarkIndex;
-            };
-
-            for (size_t i = 0; i < v.size(); ++i) {
-                REQUIRE(v[i] == runs);
-            }
-        }
-    }
-
-    SECTION("with generator") {
-        auto generated = GENERATE(range(0, 10));
-        BENCHMARK("Fill vector generated") {
-            v = std::vector<int>();
-            v.resize(size);
-            for (int i = 0; i < size; ++i)
-                v[i] = generated;
-        };
-        for (size_t i = 0; i < v.size(); ++i) {
-            REQUIRE(v[i] == generated);
-        }
-    }
-}