diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index b9fdceab..6c982790 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -923,7 +923,7 @@ MiscTests.cpp: MiscTests.cpp:: PASSED: - REQUIRE_THAT( "", Equals(nullptr) ) + REQUIRE_THAT( "", Equals(0) ) with expansion: "" equals: "" @@ -5446,9 +5446,9 @@ ConditionTests.cpp: ConditionTests.cpp:: PASSED: - REQUIRE( p == nullptr ) + REQUIRE( p == 0 ) with expansion: - NULL == nullptr + NULL == 0 ConditionTests.cpp:: PASSED: @@ -5458,39 +5458,39 @@ with expansion: ConditionTests.cpp:: PASSED: - REQUIRE( p != nullptr ) + REQUIRE( p != 0 ) with expansion: - 0x != nullptr + 0x != 0 ConditionTests.cpp:: PASSED: - REQUIRE( cp != nullptr ) + REQUIRE( cp != 0 ) with expansion: - 0x != nullptr + 0x != 0 ConditionTests.cpp:: PASSED: - REQUIRE( cpc != nullptr ) + REQUIRE( cpc != 0 ) with expansion: - 0x != nullptr + 0x != 0 ConditionTests.cpp:: PASSED: - REQUIRE( returnsNull() == nullptr ) + REQUIRE( returnsNull() == 0 ) with expansion: - {null string} == nullptr + {null string} == 0 ConditionTests.cpp:: PASSED: - REQUIRE( returnsConstNull() == nullptr ) + REQUIRE( returnsConstNull() == 0 ) with expansion: - {null string} == nullptr + {null string} == 0 ConditionTests.cpp:: PASSED: - REQUIRE( nullptr != p ) + REQUIRE( 0 != p ) with expansion: - nullptr != 0x + 0 != 0x ------------------------------------------------------------------------------- Pointers can be converted to strings @@ -7886,9 +7886,9 @@ TrickyTests.cpp: TrickyTests.cpp:: PASSED: - REQUIRE( obj.prop != nullptr ) + REQUIRE( obj.prop != 0 ) with expansion: - 0x != nullptr + 0x != 0 ------------------------------------------------------------------------------- checkedElse @@ -8253,13 +8253,13 @@ MiscTests.cpp: MiscTests.cpp:: PASSED: - REQUIRE( makeString( false ) != static_cast(nullptr) ) + REQUIRE( makeString( false ) != static_cast(0) ) with expansion: "valid string" != {null string} MiscTests.cpp:: PASSED: - REQUIRE( makeString( true ) == static_cast(nullptr) ) + REQUIRE( makeString( true ) == static_cast(0) ) with expansion: {null string} == {null string} @@ -8271,9 +8271,9 @@ TrickyTests.cpp: TrickyTests.cpp:: PASSED: - REQUIRE( ptr.get() == nullptr ) + REQUIRE( ptr.get() == 0 ) with expansion: - NULL == nullptr + NULL == 0 ------------------------------------------------------------------------------- pair > -> toString @@ -8797,18 +8797,18 @@ with expansion: "{ 0 }" == "{ 0 }" ------------------------------------------------------------------------------- -tuple +tuple<0,int,const char *> ------------------------------------------------------------------------------- ToStringTuple.cpp: ............................................................................... ToStringTuple.cpp:: PASSED: - CHECK( "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) ) + CHECK( "{ 0, 42, \"Catch me\" }" == Catch::toString(value) ) with expansion: - "{ nullptr, 42, "Catch me" }" + "{ 0, 42, "Catch me" }" == - "{ nullptr, 42, "Catch me" }" + "{ 0, 42, "Catch me" }" ------------------------------------------------------------------------------- tuple diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 02ca9f68..490c810e 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -661,7 +661,7 @@ TrickyTests.cpp: - + diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 3dd64def..b031e4d9 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -1015,7 +1015,7 @@ - "", Equals(nullptr) + "", Equals(0) "" equals: "" @@ -5709,10 +5709,10 @@ four" - p == nullptr + p == 0 - NULL == nullptr + NULL == 0 @@ -5725,50 +5725,50 @@ four" - p != nullptr + p != 0 - 0x != nullptr + 0x != 0 - cp != nullptr + cp != 0 - 0x != nullptr + 0x != 0 - cpc != nullptr + cpc != 0 - 0x != nullptr + 0x != 0 - returnsNull() == nullptr + returnsNull() == 0 - {null string} == nullptr + {null string} == 0 - returnsConstNull() == nullptr + returnsConstNull() == 0 - {null string} == nullptr + {null string} == 0 - nullptr != p + 0 != p - nullptr != 0x + 0 != 0x @@ -8318,10 +8318,10 @@ there" - obj.prop != nullptr + obj.prop != 0 - 0x != nullptr + 0x != 0 @@ -8709,7 +8709,7 @@ there" - makeString( false ) != static_cast<char*>(nullptr) + makeString( false ) != static_cast<char*>(0) "valid string" != {null string} @@ -8717,7 +8717,7 @@ there" - makeString( true ) == static_cast<char*>(nullptr) + makeString( true ) == static_cast<char*>(0) {null string} == {null string} @@ -8728,10 +8728,10 @@ there" - ptr.get() == nullptr + ptr.get() == 0 - NULL == nullptr + NULL == 0 @@ -9264,15 +9264,15 @@ there" - + - "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) + "{ 0, 42, \"Catch me\" }" == Catch::toString(value) - "{ nullptr, 42, "Catch me" }" + "{ 0, 42, "Catch me" }" == -"{ nullptr, 42, "Catch me" }" +"{ 0, 42, "Catch me" }" diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index 2fe3245d..f665c317 100644 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -1,116 +1,129 @@ -from __future__ import print_function +from __future__ import print_function import os import sys import subprocess import re +import scriptCommon from scriptCommon import catchPath -rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' ) +rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines') -filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' ) -filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' ) -pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' ) -lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' ) -hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' ) -durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' ) -versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' ) -devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' ) +filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)') +filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)') +pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)') +lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)') +hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)') +durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)') +versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)') +devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)') +nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)') if len(sys.argv) == 2: - cmdPath = sys.argv[1] + cmdPath = sys.argv[1] else: - cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) -# cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' ) + cmdPath = scriptCommon.getBuildExecutable() + +if not cmdPath.startswith("/"): + cmdPath = os.path.join(catchPath, cmdPath) overallResult = 0 -def filterLine( line ): - m = filenameParser.match( line ) - if m: - line = m.group(2) + m.group(3) - m2 = filelineParser.match( line ) - if m2: - line = m2.group(1) + "" + m2.group(3) - else: - m2 = lineNumberParser.match( line ) - if m2: - line = m2.group(1) + m2.group(2) - m = pathParser.match( line ) - if m: - path = "/" + m.group(2) - if path.startswith( catchPath ): - path = path[1+len(catchPath):] - line = m.group(1) + path + m.group(3) - m = devVersionParser.match( line ) - if m: - line = m.group(1) + "" + m.group(2) - else: - m = versionParser.match( line ) - if m: - line = m.group(1) + "" + m.group(2) +def filterLine(line): + m = filenameParser.match(line) + if m: + line = m.group(2) + m.group(3) + m2 = filelineParser.match(line) + if m2: + line = m2.group(1) + "" + m2.group(3) + else: + m2 = lineNumberParser.match(line) + if m2: + line = m2.group(1) + m2.group(2) + m = pathParser.match(line) + if m: + path = "/" + m.group(2) + if path.startswith(catchPath): + path = path[1 + len(catchPath):] + line = m.group(1) + path + m.group(3) + m = devVersionParser.match(line) + if m: + line = m.group(1) + "" + m.group(2) + else: + m = versionParser.match(line) + if m: + line = m.group(1) + "" + m.group(2) - while True: - m = hexParser.match( line ) - if m: - line = m.group(1) + "0x" + m.group(3) - else: - break - m = durationsParser.match( line ) - if m: - line = m.group(1) + 'time="{duration}"' + m.group(2) - return line + m = nullParser.match(line) + if m: + line = m.group(1) + "0" + m.group(3) -def approve( baseName, args ): - global overallResult - args[0:0] = [cmdPath] - if not os.path.exists( cmdPath ): - raise Exception( "Executable doesn't exist at " + cmdPath ) - baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) ) - rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) - filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) ) + while True: + m = hexParser.match(line) + if m: + line = m.group(1) + "0x" + m.group(3) + else: + break + m = durationsParser.match(line) + if m: + line = m.group(1) + 'time="{duration}"' + m.group(2) + return line - f = open( rawResultsPath, 'w' ) - subprocess.call( args, stdout=f, stderr=f ) - f.close() - rawFile = open( rawResultsPath, 'r' ) - filteredFile = open( filteredResultsPath, 'w' ) - for line in rawFile: - filteredFile.write( filterLine( line ).rstrip() + "\n" ) - filteredFile.close() - rawFile.close() +def approve(baseName, args): + global overallResult + args[0:0] = [cmdPath] + if not os.path.exists(cmdPath): + raise Exception("Executable doesn't exist at " + cmdPath) + baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName)) + rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName)) + filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName)) - os.remove( rawResultsPath ) - print() - print( baseName + ":" ) - if os.path.exists( baselinesPath ): - diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] ) - if diffResult == 0: - os.remove( filteredResultsPath ) - print( " \033[92mResults matched" ) - else: - print( " \n****************************\n \033[91mResults differed" ) - if diffResult > overallResult: - overallResult = diffResult - print( "\033[0m" ) - else: - print( " first approval" ) - if overallResult == 0: - overallResult = 1 + f = open(rawResultsPath, 'w') + subprocess.call(args, stdout=f, stderr=f) + f.close() + + rawFile = open(rawResultsPath, 'r') + filteredFile = open(filteredResultsPath, 'w') + for line in rawFile: + filteredFile.write(filterLine(line).rstrip() + "\n") + filteredFile.close() + rawFile.close() + + os.remove(rawResultsPath) + print() + print(baseName + ":") + if os.path.exists(baselinesPath): + diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath]) + if diffResult == 0: + os.remove(filteredResultsPath) + print(" \033[92mResults matched") + else: + print(" \n****************************\n \033[91mResults differed") + if diffResult > overallResult: + overallResult = diffResult + print("\033[0m") + else: + print(" first approval") + if overallResult == 0: + overallResult = 1 + + +print("Running approvals against executable:") +print(" " + cmdPath) # Standard console reporter -approve( "console.std", ["~_", "--order", "lex"] ) +approve("console.std", ["~_", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] ) +approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] ) +approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] ) +approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] ) +approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: - print( "run approve.py to approve new baselines" ) -exit( overallResult) + print("If these differenecs are expected run approve.py to approve new baselines") +exit(overallResult) diff --git a/scripts/scriptCommon.py b/scripts/scriptCommon.py index 6ac381ad..9adadb6d 100644 --- a/scripts/scriptCommon.py +++ b/scripts/scriptCommon.py @@ -2,8 +2,13 @@ import os import sys import subprocess + catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0]))) +def getBuildExecutable(): + dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest") + return dir + def runAndCapture( args ): child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE) lines = [] @@ -18,4 +23,4 @@ def runAndCapture( args ): line = "" else: line = line + out - return lines \ No newline at end of file + return lines