Unconditionally provide <chrono> StringMakers

When running tests in parallel, CTest runs the tests in decreasing
order of cost (time required), to get the largest speed up from
parallelism. However, the initial cost estimates for all tests are
0, and they are only updated after a test run. This works on a dev
machine, where the tests are ran over and over again, because
eventually the estimates become quite precise, but CI always does
a clean build with 0 estimates.

Because we have 2 slow tests, we want them to run first to avoid
losing parallelism. To do this, we provide them with a cost estimate
manually.

docs/configuration.md

@@ -206,7 +206,6 @@ By default, Catch does not stringify some types from the standard library. This
 
     CATCH_CONFIG_ENABLE_PAIR_STRINGMAKER     // Provide StringMaker specialization for std::pair
     CATCH_CONFIG_ENABLE_TUPLE_STRINGMAKER    // Provide StringMaker specialization for std::tuple
-    CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER   // Provide StringMaker specialization for std::chrono::duration, std::chrono::timepoint
     CATCH_CONFIG_ENABLE_VARIANT_STRINGMAKER  // Provide StringMaker specialization for std::variant, std::monostate (on C++17)
     CATCH_CONFIG_ENABLE_OPTIONAL_STRINGMAKER // Provide StringMaker specialization for std::optional (on C++17)
     CATCH_CONFIG_ENABLE_ALL_STRINGMAKERS     // Defines all of the above

docs/release-notes.md

@@ -59,7 +59,8 @@
 ### Other changes
 * `CATCH_CONFIG_DISABLE_MATCHERS` no longer exists.
   * If you do not want to use Matchers in a TU, do not include their header.
-
+* `CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER` no longer exists.
+  * `StringMaker` specializations for <chrono> are always provided
 
 ## 2.10.2
 

src/catch2/catch.hpp

@@ -60,6 +60,4 @@
 
 #include <catch2/catch_reenable_warnings.h>
 
-using Catch::Detail::Approx;
-
 #endif // TWOBLUECUBES_CATCH_HPP_INCLUDED

src/catch2/catch_approx.cpp

@@ -23,7 +23,6 @@ bool marginComparison(double lhs, double rhs, double margin) {
 }
 
 namespace Catch {
-namespace Detail {
 
     Approx::Approx ( double value )
     :   m_epsilon( std::numeric_limits<float>::epsilon()*100 ),
@@ -70,18 +69,16 @@ namespace Detail {
         m_epsilon = newEpsilon;
     }
 
-} // end namespace Detail
-
 namespace literals {
-    Detail::Approx operator "" _a(long double val) {
-        return Detail::Approx(val);
+    Approx operator "" _a(long double val) {
+        return Approx(val);
     }
-    Detail::Approx operator "" _a(unsigned long long val) {
-        return Detail::Approx(val);
+    Approx operator "" _a(unsigned long long val) {
+        return Approx(val);
     }
 } // end namespace literals
 
-std::string StringMaker<Catch::Detail::Approx>::convert(Catch::Detail::Approx const& value) {
+std::string StringMaker<Catch::Approx>::convert(Catch::Approx const& value) {
     return value.toString();
 }
 

src/catch2/catch_approx.h

@@ -13,7 +13,6 @@
 #include <type_traits>
 
 namespace Catch {
-namespace Detail {
 
     class Approx {
     private:
@@ -115,16 +114,15 @@ namespace Detail {
         double m_scale;
         double m_value;
     };
-} // end namespace Detail
 
 namespace literals {
-    Detail::Approx operator "" _a(long double val);
-    Detail::Approx operator "" _a(unsigned long long val);
+    Approx operator "" _a(long double val);
+    Approx operator "" _a(unsigned long long val);
 } // end namespace literals
 
 template<>
-struct StringMaker<Catch::Detail::Approx> {
-    static std::string convert(Catch::Detail::Approx const& value);
+struct StringMaker<Catch::Approx> {
+    static std::string convert(Catch::Approx const& value);
 };
 
 } // end namespace Catch

src/catch2/catch_decomposer.h

@@ -28,7 +28,7 @@
 #  pragma clang diagnostic ignored "-Wsign-compare"
 #elif defined __GNUC__
 #  pragma GCC diagnostic push
-#  pragma clang diagnostic ignored "-Wsign-compare"
+#  pragma GCC diagnostic ignored "-Wsign-compare"
 #endif
 
 namespace Catch {

src/catch2/catch_matchers_vector.h

@@ -124,7 +124,7 @@ namespace Matchers {
             }
 
             std::vector<T> const& m_comparator;
-            mutable Catch::Detail::Approx approx = Catch::Detail::Approx::custom();
+            mutable Catch::Approx approx = Catch::Approx::custom();
         };
 
         template<typename T>

src/catch2/catch_tostring.cpp

@@ -12,10 +12,6 @@
 #    pragma clang diagnostic ignored "-Wglobal-constructors"
 #endif
 
-// Enable specific decls locally
-#if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER)
-#define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
-#endif
 
 #include <catch2/catch_tostring.h>
 #include <catch2/catch_interfaces_config.h>
@@ -250,13 +246,6 @@ std::string StringMaker<double>::convert(double value) {
     return fpToString(value, precision);
 }
 
-std::string ratio_string<std::atto>::symbol() { return "a"; }
-std::string ratio_string<std::femto>::symbol() { return "f"; }
-std::string ratio_string<std::pico>::symbol() { return "p"; }
-std::string ratio_string<std::nano>::symbol() { return "n"; }
-std::string ratio_string<std::micro>::symbol() { return "u"; }
-std::string ratio_string<std::milli>::symbol() { return "m"; }
-
 } // end namespace Catch
 
 #if defined(__clang__)

src/catch2/catch_tostring.h

@@ -351,13 +351,12 @@ namespace Catch {
 
 //////////////////////////////////////////////////////
 // Separate std-lib types stringification, so it can be selectively enabled
-// This means that we do not bring in
+// This means that we do not bring in their headers
 
 #if defined(CATCH_CONFIG_ENABLE_ALL_STRINGMAKERS)
 #  define CATCH_CONFIG_ENABLE_PAIR_STRINGMAKER
 #  define CATCH_CONFIG_ENABLE_TUPLE_STRINGMAKER
 #  define CATCH_CONFIG_ENABLE_VARIANT_STRINGMAKER
-#  define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
 #  define CATCH_CONFIG_ENABLE_OPTIONAL_STRINGMAKER
 #endif
 
@@ -532,7 +531,6 @@ namespace Catch {
 } // namespace Catch
 
 // Separate std::chrono::duration specialization
-#if defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER)
 #include <ctime>
 #include <ratio>
 #include <chrono>
@@ -552,29 +550,31 @@ std::string ratio_string<Ratio>::symbol() {
         << Ratio::den << ']';
     return rss.str();
 }
+
+
 template <>
 struct ratio_string<std::atto> {
-    static std::string symbol();
+    static std::string symbol() { return "a"; }
 };
 template <>
 struct ratio_string<std::femto> {
-    static std::string symbol();
+    static std::string symbol() { return "f"; }
 };
 template <>
 struct ratio_string<std::pico> {
-    static std::string symbol();
+    static std::string symbol() { return "p"; }
 };
 template <>
 struct ratio_string<std::nano> {
-    static std::string symbol();
+    static std::string symbol() { return "n"; }
 };
 template <>
 struct ratio_string<std::micro> {
-    static std::string symbol();
+    static std::string symbol() { return "u"; }
 };
 template <>
 struct ratio_string<std::milli> {
-    static std::string symbol();
+    static std::string symbol() { return "m"; }
 };
 
     ////////////
@@ -647,7 +647,6 @@ struct ratio_string<std::milli> {
         }
     };
 }
-#endif // CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
 
 #include <catch2/catch_interfaces_registry_hub.h>
 

tests/CMakeLists.txt

@@ -187,7 +187,12 @@ set_tests_properties(FilteredSection-2 PROPERTIES FAIL_REGULAR_EXPRESSION "No te
 
 # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
 add_test(NAME ApprovalTests COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
-set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
+set_tests_properties(ApprovalTests
+  PROPERTIES
+    FAIL_REGULAR_EXPRESSION "Results differed"
+    COST 120 # We know that this is either the most, or second most,
+    # expensive test in the test suite, so we give it high estimate for CI runs
+)
 
 add_test(NAME RegressionCheck-1670 COMMAND $<TARGET_FILE:SelfTest> "#1670 regression check" -c A -r compact)
 set_tests_properties(RegressionCheck-1670 PROPERTIES PASS_REGULAR_EXPRESSION "Passed 1 test case with 2 assertions.")

tests/ExtraTests/CMakeLists.txt

@@ -128,6 +128,8 @@ set_tests_properties(
     BenchmarkingMacros
   PROPERTIES
     PASS_REGULAR_EXPRESSION "benchmark name                                  samples       iterations    estimated"
+    COST 80 # We know that this is either the most, or second most,
+   # expensive test in the test suite, so we give it high estimate for CI runs
 )
 
 # This test touches windows.h, so it should only be compiled under msvc

tests/SelfTest/Baselines/compact.sw.approved.txt

@@ -1108,7 +1108,7 @@ CmdLine.tests.cpp:<line number>: passed: config.benchmarkSamples == 200 for: 200
 CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-resamples=20000" }) for: {?}
 CmdLine.tests.cpp:<line number>: passed: config.benchmarkResamples == 20000 for: 20000 (0x<hex digits>) == 20000 (0x<hex digits>)
 CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) for: {?}
-CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) for: 0.99 == Approx( 0.99 )
+CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Approx(0.99) for: 0.99 == Approx( 0.99 )
 CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
 CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true
 Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1

tests/SelfTest/Baselines/console.sw.approved.txt

@@ -8022,7 +8022,7 @@ with expansion:
   {?}
 
 CmdLine.tests.cpp:<line number>: PASSED:
-  REQUIRE( config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) )
+  REQUIRE( config.benchmarkConfidenceInterval == Catch::Approx(0.99) )
 with expansion:
   0.99 == Approx( 0.99 )
 

tests/SelfTest/Baselines/xml.sw.approved.txt

@@ -10103,7 +10103,7 @@ Nor would this
           </Expression>
           <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
             <Original>
-              config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99)
+              config.benchmarkConfidenceInterval == Catch::Approx(0.99)
             </Original>
             <Expanded>
               0.99 == Approx( 0.99 )

tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp

@@ -489,7 +489,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
         SECTION("resamples") {
             CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
 
-            REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
+            REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99));
         }
 
         SECTION("resamples") {

tests/SelfTest/IntrospectiveTests/GeneratorsImpl.tests.cpp

@@ -178,7 +178,7 @@ TEST_CASE("Generators internals", "[generators][internals]") {
             }
 
             SECTION("Floating Point") {
-                using Catch::Detail::Approx;
+                using Catch::Approx;
                 SECTION("Exact") {
                     const auto rangeStart = -1.;
                     const auto rangeEnd = 1.;

tests/SelfTest/UsageTests/Approx.tests.cpp

@@ -11,7 +11,7 @@
 
 #include <cmath>
 
-using Catch::Detail::Approx;
+using Catch::Approx;
 
 namespace { namespace ApproxTests {
 

tests/SelfTest/UsageTests/Condition.tests.cpp

@@ -14,7 +14,7 @@
 #include <catch2/catch_approx.h>
 #include <catch2/catch_test_macros.hpp>
 
-using Catch::Detail::Approx;
+using Catch::Approx;
 
 #include <string>
 #include <limits>

tests/SelfTest/UsageTests/ToStringChrono.tests.cpp

@@ -1,4 +1,3 @@
-#define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
 #include <catch2/catch_test_macros.hpp>
 
 #include <chrono>

tools/scripts/approvalTests.py

@@ -70,6 +70,8 @@ nanParser = re.compile(r'''
     __builtin_nanf\("0x<hex\ digits>"\)  # The weird content of the brackets is there because a different parser has already ran before this one
 ''', re.VERBOSE)
 
+# The weird OR is there to always have at least empty string for group 1
+tapTestNumParser = re.compile(r'^(not |)?ok (\d+) -')
 
 if len(sys.argv) == 2:
     cmdPath = sys.argv[1]
@@ -125,6 +127,9 @@ def filterLine(line, isCompact):
         line = line.replace(': FAILED', ': failed')
         line = line.replace(': PASSED', ': passed')
 
+    # strip out the test order number in TAP to avoid massive diffs for every change
+    line = tapTestNumParser.sub("\g<1>ok {test-number} -", line)
+
     # strip Catch version number
     line = versionParser.sub("<version>", line)