mirror of
https://github.com/catchorg/Catch2.git
synced 2025-04-29 20:13:52 +00:00
3087e19cc7
This greatly simplifies running Catch2 tests in single binary in parallel from external test runners. Instead of having to shard the tests by tags/test names, an external test runner can now just ask for test shard 2 (out of X), and execute that in single process, without having to know what tests are actually in the shard. Note that sharding also applies to test listing, and happens after tests were ordered according to the `--order` feature.
91 lines
2.7 KiB
Python
91 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""
|
|
This test script verifies that sharding tests does change which tests are run.
|
|
This is done by running the binary multiple times, once to list all the tests,
|
|
once per shard to list the tests for that shard, and once again per shard to
|
|
execute the tests. The sharded lists are compared to the full list to ensure
|
|
none are skipped, duplicated, and that the order remains the same. This process
|
|
is repeated for multiple command line argument combinations to ensure sharding
|
|
works with different filters and test orderings.
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
from collections import namedtuple
|
|
|
|
def make_base_commandline(self_test_exe):
|
|
return [
|
|
self_test_exe,
|
|
'--reporter', 'xml',
|
|
"--shard-count", "5",
|
|
"--shard-index", "2",
|
|
"[generators]~[benchmarks]~[.]"
|
|
]
|
|
|
|
def list_tests(self_test_exe):
|
|
cmd = make_base_commandline(self_test_exe) + ['--list-tests']
|
|
|
|
process = subprocess.Popen(
|
|
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
stdout, stderr = process.communicate()
|
|
if stderr:
|
|
raise RuntimeError("Unexpected error output:\n" + process.stderr)
|
|
|
|
root = ET.fromstring(stdout)
|
|
result = [elem.text for elem in root.findall('./TestCase/Name')]
|
|
|
|
if len(result) < 2:
|
|
raise RuntimeError("Unexpectedly few tests listed (got {})".format(
|
|
len(result)))
|
|
|
|
return result
|
|
|
|
|
|
def execute_tests(self_test_exe):
|
|
cmd = make_base_commandline(self_test_exe)
|
|
|
|
process = subprocess.Popen(
|
|
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
stdout, stderr = process.communicate()
|
|
if stderr:
|
|
raise RuntimeError("Unexpected error output:\n" + process.stderr)
|
|
|
|
root = ET.fromstring(stdout)
|
|
result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
|
|
|
|
if len(result) < 2:
|
|
raise RuntimeError("Unexpectedly few tests listed (got {})".format(
|
|
len(result)))
|
|
return result
|
|
|
|
|
|
def check_listed_and_executed_tests_match(listed_tests, executed_tests):
|
|
listed_names = set(listed_tests)
|
|
executed_names = set(executed_tests)
|
|
|
|
listed_string = "\n".join(listed_names)
|
|
exeucted_string = "\n".join(executed_names)
|
|
|
|
assert listed_names == executed_names, (
|
|
"Executed tests do not match the listed tests:\nExecuted:\n{}\n\nListed:\n{}".format(exeucted_string, listed_string)
|
|
)
|
|
|
|
|
|
def test_sharding(self_test_exe):
|
|
listed_tests = list_tests(self_test_exe)
|
|
executed_tests = execute_tests(self_test_exe)
|
|
|
|
check_listed_and_executed_tests_match(listed_tests, executed_tests)
|
|
|
|
|
|
def main():
|
|
self_test_exe, = sys.argv[1:]
|
|
|
|
test_sharding(self_test_exe)
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|