From 13bfe477c7d4d73a8988357f7a492c8c4491ab1b Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Fri, 11 Mar 2011 19:44:59 +0000 Subject: [PATCH] Improvements to self test to give finer grained feedback --- Test/TestMain.cpp | 51 +++++++++++---- internal/catch_common.h | 12 ++++ internal/catch_interfaces_runner.h | 2 + internal/catch_interfaces_testcase.h | 4 ++ internal/catch_runner_impl.hpp | 38 ------------ internal/catch_self_test.hpp | 72 +++++++++++++++++++++- internal/catch_test_case_info.hpp | 38 ++++++++++++ internal/catch_test_case_registry_impl.hpp | 23 +++++++ 8 files changed, 190 insertions(+), 50 deletions(-) diff --git a/Test/TestMain.cpp b/Test/TestMain.cpp index 7ddcc23c..cb6d327f 100644 --- a/Test/TestMain.cpp +++ b/Test/TestMain.cpp @@ -11,22 +11,51 @@ */ #include "../catch_with_main.hpp" - #include "../internal/catch_self_test.hpp" TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results" ) { - Catch::EmbeddedRunner runner; + using namespace Catch; + + /////////////////////////////////////////////////////////////////////////// + SECTION( "selftest/expected result", + "Tests do what they claim" ) + { + SECTION( "selftest/expected result/failing tests", + "Tests in the 'failing' branch fail" ) + { + MetaTestRunner::runMatching( "./failing/*", MetaTestRunner::Expected::ToFail ); + } + + SECTION( "selftest/expected result/succeeding tests", + "Tests in the 'succeeding' branch succeed" ) + { + MetaTestRunner::runMatching( "./succeeding/*", MetaTestRunner::Expected::ToSucceed ); + } + } + + /////////////////////////////////////////////////////////////////////////// + SECTION( "selftest/test counts", + "Number of test cases that run is fixed" ) + { + EmbeddedRunner runner; + + SECTION( "selftest/test counts/succeeding tests", + "Number of 'succeeding' tests is fixed" ) + { + runner.runMatching( "./succeeding/*" ); + CHECK( runner.getSuccessCount() == 213 ); + CHECK( runner.getFailureCount() == 0 ); + } - runner.runMatching( "./succeeding/*" ); - INFO( runner.getOutput() ); - CHECK( runner.getSuccessCount() == 213 ); - CHECK( runner.getFailureCount() == 0 ); - - runner.runMatching( "./failing/*" ); - INFO( runner.getOutput() ); - CHECK( runner.getSuccessCount() == 0 ); - CHECK( runner.getFailureCount() == 54 ); + SECTION( "selftest/test counts/failing tests", + "Number of 'failing' tests is fixed" ) + { + runner.runMatching( "./failing/*" ); + CHECK( runner.getSuccessCount() == 0 ); + CHECK( runner.getFailureCount() == 54 ); + } + } } TEST_CASE( "meta/Misc/Sections", "looped tests" ) diff --git a/internal/catch_common.h b/internal/catch_common.h index 3282044d..02307d2d 100644 --- a/internal/catch_common.h +++ b/internal/catch_common.h @@ -63,6 +63,18 @@ namespace Catch } } + template + inline void forEach( ContainerT& container, Function function ) + { + std::for_each( container.begin(), container.end(), function ); + } + + template + inline void forEach( const ContainerT& container, Function function ) + { + std::for_each( container.begin(), container.end(), function ); + } + ATTRIBUTE_NORETURN inline void throwLogicError( const std::string& message, const std::string& file, long line ) { diff --git a/internal/catch_interfaces_runner.h b/internal/catch_interfaces_runner.h index 864ce634..05d82913 100644 --- a/internal/catch_interfaces_runner.h +++ b/internal/catch_interfaces_runner.h @@ -16,6 +16,8 @@ namespace Catch { + class TestCaseInfo; + struct IRunner { virtual ~IRunner diff --git a/internal/catch_interfaces_testcase.h b/internal/catch_interfaces_testcase.h index 0bede865..87bc537f 100644 --- a/internal/catch_interfaces_testcase.h +++ b/internal/catch_interfaces_testcase.h @@ -52,6 +52,10 @@ namespace Catch virtual const std::vector& getAllTests () const = 0; + + virtual std::vector getMatchingTestCases + ( const std::string& rawTestSpec + ) = 0; }; } diff --git a/internal/catch_runner_impl.hpp b/internal/catch_runner_impl.hpp index 657682df..0593259f 100644 --- a/internal/catch_runner_impl.hpp +++ b/internal/catch_runner_impl.hpp @@ -56,44 +56,6 @@ namespace Catch std::string& m_targetString; }; - /////////////////////////////////////////////////////////////////////////// - /////////////////////////////////////////////////////////////////////////// - - class TestSpec - { - public: - /////////////////////////////////////////////////////////////////////// - TestSpec - ( - const std::string& rawSpec - ) - : m_rawSpec( rawSpec ), - m_isWildcarded( false ) - { - if( m_rawSpec[m_rawSpec.size()-1] == '*' ) - { - m_rawSpec = m_rawSpec.substr( 0, m_rawSpec.size()-1 ); - m_isWildcarded = true; - } - } - - /////////////////////////////////////////////////////////////////////// - bool matches - ( - const std::string& testName - ) - const - { - if( !m_isWildcarded ) - return m_rawSpec == testName; - else - return testName.size() >= m_rawSpec.size() && testName.substr( 0, m_rawSpec.size() ) == m_rawSpec; - } - - private: - std::string m_rawSpec; - bool m_isWildcarded; - }; /////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////// diff --git a/internal/catch_self_test.hpp b/internal/catch_self_test.hpp index 646b7e37..efee848e 100644 --- a/internal/catch_self_test.hpp +++ b/internal/catch_self_test.hpp @@ -50,7 +50,7 @@ namespace Catch m_output = oss.str(); return result; } - + /////////////////////////////////////////////////////////////////////////// std::string getOutput () @@ -79,6 +79,76 @@ namespace Catch std::size_t m_failures; std::string m_output; }; + + class MetaTestRunner + { + public: + struct Expected + { + enum Result + { + ToSucceed, + ToFail + }; + }; + + /////////////////////////////////////////////////////////////////////////// + static void runMatching + ( + const std::string& testSpec, + Expected::Result expectedResult + ) + { + forEach( Hub::getTestCaseRegistry().getMatchingTestCases( testSpec ), + MetaTestRunner( expectedResult ) ); + } + + /////////////////////////////////////////////////////////////////////////// + MetaTestRunner + ( + Expected::Result expectedResult + ) + : m_expectedResult( expectedResult ) + { + } + + /////////////////////////////////////////////////////////////////////////// + void operator() + ( + const TestCaseInfo& testCase + ) + { + EmbeddedRunner runner; + runner.runMatching( testCase.getName() ); + switch( m_expectedResult ) + { + case Expected::ToSucceed: + if( runner.getFailureCount() > 0 ) + { + INFO( runner.getOutput() ); + FAIL( "Expected test case '" + << testCase.getName() + << "' to succeed but there was/ were " + << runner.getFailureCount() << " failure(s)" ); + } + break; + case Expected::ToFail: + if( runner.getSuccessCount() > 0 ) + { + INFO( runner.getOutput() ); + FAIL( "Expected test case '" + << testCase.getName() + << "' to fail but there was/ were " + << runner.getSuccessCount() << " success(es)" ); + } + break; + } + }; + + private: + Expected::Result m_expectedResult; + }; + } #endif // TWOBLUECUBES_CATCH_SELF_TEST_HPP_INCLUDED diff --git a/internal/catch_test_case_info.hpp b/internal/catch_test_case_info.hpp index f004b0f5..9c796f5f 100644 --- a/internal/catch_test_case_info.hpp +++ b/internal/catch_test_case_info.hpp @@ -157,6 +157,44 @@ namespace Catch std::string m_description; }; + /////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////// + + class TestSpec + { + public: + /////////////////////////////////////////////////////////////////////// + TestSpec + ( + const std::string& rawSpec + ) + : m_rawSpec( rawSpec ), + m_isWildcarded( false ) + { + if( m_rawSpec[m_rawSpec.size()-1] == '*' ) + { + m_rawSpec = m_rawSpec.substr( 0, m_rawSpec.size()-1 ); + m_isWildcarded = true; + } + } + + /////////////////////////////////////////////////////////////////////// + bool matches + ( + const std::string& testName + ) + const + { + if( !m_isWildcarded ) + return m_rawSpec == testName; + else + return testName.size() >= m_rawSpec.size() && testName.substr( 0, m_rawSpec.size() ) == m_rawSpec; + } + + private: + std::string m_rawSpec; + bool m_isWildcarded; + }; } #endif // TWOBLUECUBES_CATCH_TESTCASEINFO_HPP_INCLUDED diff --git a/internal/catch_test_case_registry_impl.hpp b/internal/catch_test_case_registry_impl.hpp index 12700308..2e430dde 100644 --- a/internal/catch_test_case_registry_impl.hpp +++ b/internal/catch_test_case_registry_impl.hpp @@ -18,6 +18,7 @@ #include #include +#include // !TBD DBG namespace Catch { class TestRegistry : public ITestCaseRegistry @@ -56,6 +57,28 @@ namespace Catch { return m_functionsInOrder; } + + /////////////////////////////////////////////////////////////////////////// + virtual std::vector getMatchingTestCases + ( + const std::string& rawTestSpec + ) + { + TestSpec testSpec( rawTestSpec ); + + std::vector testList; + std::vector::const_iterator it = m_functionsInOrder.begin(); + std::vector::const_iterator itEnd = m_functionsInOrder.end(); + for(; it != itEnd; ++it ) + { + if( testSpec.matches( it->getName() ) ) + { + testList.push_back( *it ); + std::cout << it->getName() << std::endl; + } + } + return testList; + } private: