Squashed 'externals/catch/' content from commit ab6c7375b

git-subtree-dir: externals/catch
git-subtree-split: ab6c7375be9a8e71ee84c6f8537113f9f47daf99

tests/TestScripts/ConfigureTestsCommon.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from typing import List, Tuple
+
+import os
+import subprocess
+
+def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
+    base_configure_cmd = ['cmake',
+                          '-B{}'.format(project_path),
+                          '-H{}'.format(source_path),
+                          '-DCMAKE_BUILD_TYPE=Debug',
+                          '-DCATCH_DEVELOPMENT_BUILD=ON']
+    for option, value in options:
+        base_configure_cmd.append('-D{}={}'.format(option, value))
+    try:
+        subprocess.run(base_configure_cmd,
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.STDOUT,
+                       check = True)
+    except subprocess.SubprocessError as ex:
+        print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
+        print("Return code: {}".format(ex.returncode))
+        print("output: {}".format(ex.output))
+        raise
+    print('Configuring {} finished'.format(project_path))
+
+    build_cmd = ['cmake',
+                 '--build', '{}'.format(project_path),
+                 # For now we assume that we only need Debug config
+                 '--config', 'Debug']
+    try:
+        subprocess.run(build_cmd,
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.STDOUT,
+                       check = True)
+    except subprocess.SubprocessError as ex:
+        print("Could not build project in '{}'".format(project_path))
+        print("Return code: {}".format(ex.returncode))
+        print("output: {}".format(ex.output))
+        raise
+    print('Building {} finished'.format(project_path))
+
+def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
+    # For now we assume that Windows builds are done using MSBuild under
+    # Debug configuration. This means that we need to add "Debug" folder
+    # to the path when constructing it. On Linux, we don't add anything.
+    config_path = "Debug" if os.name == 'nt' else ""
+    full_path = os.path.join(base_path, config_path, binary_name)
+
+    base_cmd = [full_path]
+    base_cmd.extend(other_options)
+
+    try:
+        ret = subprocess.run(base_cmd,
+                             stdout = subprocess.PIPE,
+                             stderr = subprocess.PIPE,
+                             check = True,
+                             universal_newlines = True)
+    except subprocess.SubprocessError as ex:
+        print('Could not run "{}"'.format(base_cmd))
+        print('Args: "{}"'.format(other_options))
+        print('Return code: {}'.format(ex.returncode))
+        print('stdout: {}'.format(ex.stdout))
+        print('stderr: {}'.format(ex.stdout))
+        raise
+
+    return (ret.stdout, ret.stderr)

tests/TestScripts/testBazelReporter.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+import os
+import re
+import sys
+import xml.etree.ElementTree as ET
+import subprocess
+
+"""
+Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
+a junit reporter that writes to the provided path.
+
+Requires 2 arguments, path to Catch2 binary configured with
+`CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
+"""
+if len(sys.argv) != 3:
+    print("Wrong number of arguments: {}".format(len(sys.argv)))
+    print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
+    exit(1)
+
+
+bin_path = os.path.abspath(sys.argv[1])
+output_dir = os.path.abspath(sys.argv[2])
+xml_out_path = os.path.join(output_dir, '{}.xml'.format(os.path.basename(bin_path)))
+
+# Ensure no file exists from previous test runs
+if os.path.isfile(xml_out_path):
+    os.remove(xml_out_path)
+
+print('bin path:', bin_path)
+print('xml out path:', xml_out_path)
+
+env = os.environ.copy()
+env["XML_OUTPUT_FILE"] = xml_out_path
+test_passing = True
+
+try:
+    ret = subprocess.run(
+        bin_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+        env=env
+    )
+    stdout = ret.stdout
+except subprocess.SubprocessError as ex:
+    if ex.returncode == 1:
+        # The test cases are allowed to fail.
+        test_passing = False
+        stdout = ex.stdout
+    else:
+        print('Could not run "{}"'.format(bin_path))
+        print("Return code: {}".format(ex.returncode))
+        print("stdout: {}".format(ex.stdout))
+        print("stderr: {}".format(ex.stderr))
+        raise
+
+# Check for valid XML output
+try:
+    tree = ET.parse(xml_out_path)
+except ET.ParseError as ex:
+    print("Invalid XML: '{}'".format(ex))
+    raise
+except FileNotFoundError as ex:
+    print("Could not find '{}'".format(xml_out_path))
+    raise
+
+bin_name = os.path.basename(bin_path)
+# Check for matching testsuite
+if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
+    print("Could not find '{}' testsuite".format(bin_name))
+    exit(2)
+
+# Check that we haven't disabled the default reporter
+summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
+if len(summary_test_cases) == 0:
+    print("Could not find test summary in {}".format(stdout))
+    exit(2)
+
+total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
+
+if failed == 0 and not test_passing:
+    print("Expected at least 1 test failure!")
+    exit(2)
+
+if len(tree.findall('.//testcase')) != total:
+    print("Unexpected number of test cases!")
+    exit(2)
+
+if len(tree.findall('.//failure')) != failed:
+    print("Unexpected number of test failures!")
+    exit(2)
+
+if (passed + failed) != total:
+    print("Something has gone very wrong, ({} + {}) != {}".format(passed, failed, total))
+    exit(2)

tests/TestScripts/testBazelSharding.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+import os
+import re
+import sys
+import subprocess
+
+"""
+Test that Catch2 recognizes the three sharding-related environment variables
+and responds accordingly (running only the selected shard, creating the
+response file, etc).
+
+Requires 2 arguments, path to Catch2 binary to run and the output directory
+for the output file.
+"""
+if len(sys.argv) != 3:
+    print("Wrong number of arguments: {}".format(len(sys.argv)))
+    print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
+    exit(1)
+
+
+bin_path = os.path.abspath(sys.argv[1])
+output_dir = os.path.abspath(sys.argv[2])
+info_file_path = os.path.join(output_dir, '{}.shard-support'.format(os.path.basename(bin_path)))
+
+# Ensure no file exists from previous test runs
+if os.path.isfile(info_file_path):
+    os.remove(info_file_path)
+
+print('bin path:', bin_path)
+print('shard confirmation path:', info_file_path)
+
+env = os.environ.copy()
+# We will run only one shard, and it should have the passing test.
+# This simplifies our work a bit, and if we have any regression in this
+# functionality we can make more complex tests later.
+env["BAZEL_TEST"] = "1"
+env["TEST_SHARD_INDEX"] = "0"
+env["TEST_TOTAL_SHARDS"] = "2"
+env["TEST_SHARD_STATUS_FILE"] = info_file_path
+
+
+try:
+    ret = subprocess.run(
+        bin_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+        env=env
+    )
+    stdout = ret.stdout
+except subprocess.SubprocessError as ex:
+    print('Could not run "{}"'.format(bin_path))
+    print("Return code: {}".format(ex.returncode))
+    print("stdout: {}".format(ex.stdout))
+    print("stderr: {}".format(ex.stderr))
+    raise
+
+
+if not "All tests passed (1 assertion in 1 test case)" in stdout:
+    print("Did not find expected output in stdout.")
+    print("stdout:\n{}".format(stdout))
+    exit(1)
+
+if not os.path.isfile(info_file_path):
+    print("Catch2 did not create expected file at path '{}'".format(info_file_path))
+    exit(2)

tests/TestScripts/testConfigureDefaultReporter.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DEFAULT_REPORTER", "xml")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
+
+xml_tag = '</Catch2TestRun>'
+if xml_tag not in stdout:
+    print("Could not find '{}' in the stdout".format(xml_tag))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureDisable.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DISABLE", "ON"),
+                     # We need to turn off WERROR, because the compilers
+                     # can see that the various variables inside test cases
+                     # are set but unused.
+                     ("CATCH_ENABLE_WERROR", "OFF")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['--allow-running-no-tests'])
+
+
+summary_line = 'No tests ran'
+if not summary_line in stdout:
+    print("Could not find '{}' in the stdout".format(summary_line))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureDisableStringification.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['-s', '[approx][custom]'])
+
+
+required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION'
+if not required_output in stdout:
+    print("Could not find '{}' in the stdout".format(required_output))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testConfigureExperimentalRedirect.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+    print('Wrong number of arguments: {}'.format(len(sys.argv)))
+    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+    exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect')
+
+configure_and_build(catch2_source_path,
+                    build_dir_path,
+                    [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                                  'SelfTest',
+                                  ['-r', 'xml', '"has printf"'])
+
+
+# The print from printf must be within the XML's reporter stdout tag.
+required_output = '''\
+      <StdOut>
+loose text artifact
+      </StdOut>
+'''
+if not required_output in stdout:
+    print("Could not find '{}' in the stdout".format(required_output))
+    print('stdout: "{}"'.format(stdout))
+    exit(2)

tests/TestScripts/testPartialTestCaseEvent.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+"""
+This test script verifies that the testCasePartial{Starting,Ended} reporter
+events fire properly. This is done by calling a test binary compiled with
+reporter that reports specifically testCase* events, and verifying the
+outputs match what we expect.
+"""
+
+import subprocess
+import sys
+
+expected_section_output = '''\
+TestCaseStarting: section
+TestCaseStartingPartial: section#0
+TestCasePartialEnded: section#0
+TestCaseStartingPartial: section#1
+TestCasePartialEnded: section#1
+TestCaseStartingPartial: section#2
+TestCasePartialEnded: section#2
+TestCaseStartingPartial: section#3
+TestCasePartialEnded: section#3
+TestCaseEnded: section
+'''
+
+expected_generator_output = '''\
+TestCaseStarting: generator
+TestCaseStartingPartial: generator#0
+TestCasePartialEnded: generator#0
+TestCaseStartingPartial: generator#1
+TestCasePartialEnded: generator#1
+TestCaseStartingPartial: generator#2
+TestCasePartialEnded: generator#2
+TestCaseStartingPartial: generator#3
+TestCasePartialEnded: generator#3
+TestCaseEnded: generator
+'''
+
+
+from typing import List
+
+def get_test_output(test_exe: str, sections: bool) -> List[str]:
+    cmd = [test_exe, '--reporter', 'partial']
+    if sections:
+        cmd.append('section')
+    else:
+        cmd.append('generator')
+
+    ret = subprocess.run(cmd,
+                         stdout = subprocess.PIPE,
+                         stderr = subprocess.PIPE,
+                         timeout = 10,
+                         check = True,
+                         universal_newlines = True)
+
+    return ret.stdout
+
+def main():
+    test_exe, = sys.argv[1:]
+    actual_section_output = get_test_output(test_exe, sections = True)
+
+    assert actual_section_output == expected_section_output, (
+    'Sections\nActual:\n{}\nExpected:\n{}\n'.format(actual_section_output, expected_section_output))
+
+    actual_generator_output = get_test_output(test_exe, sections = False)
+    assert actual_generator_output == expected_generator_output, (
+    'Generators\nActual:\n{}\nExpected:\n{}\n'.format(actual_generator_output, expected_generator_output))
+
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/TestScripts/testRandomOrder.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+"""
+This test script verifies that the random ordering of tests inside
+Catch2 is invariant in regards to subsetting. This is done by running
+the binary 3 times, once with all tests selected, and twice with smaller
+subsets of tests selected, and verifying that the selected tests are in
+the same relative order.
+"""
+
+import subprocess
+import sys
+import random
+import xml.etree.ElementTree as ET
+
+def none_to_empty_str(e):
+    if e is None:
+        return ""
+    assert type(e) is str
+    return e
+
+def list_tests(self_test_exe, tags, rng_seed):
+    cmd = [self_test_exe, '--reporter', 'xml', '--list-tests', '--order', 'rand',
+            '--rng-seed', str(rng_seed)]
+    tags_arg = ','.join('[{}]~[.]'.format(t) for t in tags)
+    if tags_arg:
+        cmd.append(tags_arg)
+    process = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    if stderr:
+        raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+    root = ET.fromstring(stdout)
+    result = [(none_to_empty_str(tc.find('Name').text),
+               none_to_empty_str(tc.find('Tags').text),
+               none_to_empty_str(tc.find('ClassName').text)) for tc in root.findall('./TestCase')]
+
+    if len(result) < 2:
+        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+            len(result)))
+    return result
+
+def check_is_sublist_of(shorter, longer):
+    assert len(shorter) < len(longer)
+    assert len(set(longer)) == len(longer)
+
+    indexes_in_longer = {s: i for i, s in enumerate(longer)}
+    for s1, s2 in zip(shorter, shorter[1:]):
+        assert indexes_in_longer[s1] < indexes_in_longer[s2], (
+                '{} comes before {} in longer list.\n'
+                'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter))
+
+def main():
+    self_test_exe, = sys.argv[1:]
+
+    # We want a random seed for the test, but want to avoid 0,
+    # because it has special meaning
+    seed = random.randint(1, 2 ** 32 - 1)
+
+    list_one_tag = list_tests(self_test_exe, ['generators'], seed)
+    list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed)
+    list_all = list_tests(self_test_exe, [], seed)
+
+    # First, verify that restricting to a subset yields the same order
+    check_is_sublist_of(list_two_tags, list_all)
+    check_is_sublist_of(list_one_tag, list_two_tags)
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/TestScripts/testSharding.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+"""
+This test script verifies that sharding tests does change which tests are run.
+This is done by running the binary multiple times, once to list all the tests,
+once per shard to list the tests for that shard, and once again per shard to
+execute the tests. The sharded lists are compared to the full list to ensure
+none are skipped, duplicated, and that the order remains the same.
+"""
+
+import random
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+from collections import namedtuple
+
+from typing import List, Dict
+
+seed = random.randint(0, 2 ** 32 - 1)
+number_of_shards = 5
+
+def make_base_commandline(self_test_exe):
+    return [
+        self_test_exe,
+        '--reporter', 'xml',
+        '--order', 'rand',
+        '--rng-seed', str(seed),
+        "[generators]~[benchmarks]~[.]"
+    ]
+
+
+def list_tests(self_test_exe: str, extra_args: List[str] = None):
+    cmd = make_base_commandline(self_test_exe) + ['--list-tests']
+    if extra_args:
+        cmd.extend(extra_args)
+
+    try:
+        ret = subprocess.run(cmd,
+                             stdout = subprocess.PIPE,
+                             stderr = subprocess.PIPE,
+                             timeout = 10,
+                             check = True,
+                             universal_newlines = True)
+    except subprocess.CalledProcessError as ex:
+        print('Could not list tests:\n{}'.format(ex.stderr))
+
+    if ret.stderr:
+        raise RuntimeError("Unexpected error output:\n" + ret.stderr)
+
+    root = ET.fromstring(ret.stdout)
+    result = [elem.text for elem in root.findall('./TestCase/Name')]
+
+    if len(result) < 2:
+        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+            len(result)))
+
+
+    return result
+
+
+def execute_tests(self_test_exe: str, extra_args: List[str] = None):
+    cmd = make_base_commandline(self_test_exe)
+    if extra_args:
+        cmd.extend(extra_args)
+
+    try:
+        ret = subprocess.run(cmd,
+                             stdout = subprocess.PIPE,
+                             stderr = subprocess.PIPE,
+                             timeout = 10,
+                             check = True,
+                             universal_newlines = True)
+    except subprocess.CalledProcessError as ex:
+        print('Could not list tests:\n{}'.format(ex.stderr))
+
+    if ret.stderr:
+        raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+    root = ET.fromstring(ret.stdout)
+    result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
+
+    if len(result) < 2:
+        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+            len(result)))
+
+    return result
+
+
+def test_sharded_listing(self_test_exe: str) -> Dict[int, List[str]]:
+    """
+    Asks the test binary for list of all tests, and also for lists of
+    tests from shards.
+
+    The combination of shards is then checked whether it corresponds to
+    the full list of all tests.
+
+    Returns the dictionary of shard-index => listed tests for later use.
+    """
+    all_tests = list_tests(self_test_exe)
+    big_shard_tests = list_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
+
+    assert all_tests == big_shard_tests, (
+        "No-sharding test list does not match the listing of big shard:\nNo shard:\n{}\n\nWith shard:\n{}\n".format(
+            '\n'.join(all_tests),
+            '\n'.join(big_shard_tests)
+        )
+    )
+
+    shard_listings = dict()
+    for shard_idx in range(number_of_shards):
+        shard_listings[shard_idx] = list_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
+
+    shard_sizes = [len(v) for v in shard_listings.values()]
+    assert len(all_tests) == sum(shard_sizes)
+
+    # Check that the shards have roughly the right sizes (e.g. we don't
+    # have all tests in single shard and the others are empty)
+    differences = [abs(x1 - x2) for x1, x2 in zip(shard_sizes, shard_sizes[1:])]
+    assert all(diff <= 1 for diff in differences), "A shard has weird size: {}".format(shard_sizes)
+
+    combined_shards = [inner for outer in shard_listings.values() for inner in outer]
+    assert all_tests == combined_shards, (
+        "All tests and combined shards disagree.\nNo shard:\n{}\n\nCombined:\n{}\n\n".format(
+            '\n'.join(all_tests),
+            '\n'.join(combined_shards)
+        )
+    )
+    shard_listings[-1] = all_tests
+
+    return shard_listings
+
+
+def test_sharded_execution(self_test_exe: str, listings: Dict[int, List[str]]):
+    """
+    Runs the test binary and checks that the executed tests match the
+    previously listed tests.
+
+    Also does this for various shard indices, and that the combination
+    of all shards matches the full run/listing.
+    """
+    all_tests = execute_tests(self_test_exe)
+    big_shard_tests = execute_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
+    assert all_tests == big_shard_tests
+
+    assert listings[-1] == all_tests
+
+    for shard_idx in range(number_of_shards):
+        assert listings[shard_idx] == execute_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
+
+
+def main():
+    self_test_exe, = sys.argv[1:]
+    listings = test_sharded_listing(self_test_exe)
+    test_sharded_execution(self_test_exe, listings)
+
+if __name__ == '__main__':
+    sys.exit(main())