Skip to content

Commit

Permalink
Reformat test-scripts with black
Browse files Browse the repository at this point in the history
  • Loading branch information
@arichardson
arichardson committed Aug 9, 2023
1 parent 28c9b88 commit e45fa1c
Show file tree
Hide file tree
Showing 20 changed files with 779 additions and 290 deletions.
11 changes: 7 additions & 4 deletions test-scripts/kyua_db_to_junit_xml.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ def convert_kyua_db_to_junit_xml(db_file: Path, output_file: Path, prefix: "Opti
def fixup_kyua_generated_junit_xml(xml_file: Path, prefix: "Optional[str]" = None):
boot_cheribsd.info("Updating statistics in JUnit file ", xml_file)
# Process junit xml file with junitparser to update the number of tests, failures, total time, etc.
orig_xml_str = xml_file.read_text("utf-8", errors='backslashreplace')
orig_xml_str = xml_file.read_text("utf-8", errors="backslashreplace")
xml_str = orig_xml_str
for i in range(32):
if chr(i) not in ("\n", "\t"):
# Can't reference NULL character -> backslashescape instead
# xml_str = xml_str.replace(chr(i), "&#" + str(i) + ";")
xml_str = xml_str.replace(chr(i), "\\x" + format(i, '02x') + ";")
xml_str = xml_str.replace(chr(i), "\\x" + format(i, "02x") + ";")
with tempfile.NamedTemporaryFile("wb") as tf:
# create a temporary file first to avoid clobbering the original one if we fail to parse it
tf.write(xml_str.encode("ascii", errors="xmlcharrefreplace"))
Expand Down Expand Up @@ -83,8 +83,11 @@ def fixup_kyua_generated_junit_xml(xml_file: Path, prefix: "Optional[str]" = Non

parser = argparse.ArgumentParser()
parser.add_argument("db", help="The database to convert")
parser.add_argument("xml", nargs=argparse.OPTIONAL,
help="The output file (or - for stdout). Defaults to the db file with suffix .xml")
parser.add_argument(
"xml",
nargs=argparse.OPTIONAL,
help="The output file (or - for stdout). Defaults to the db file with suffix .xml",
)
parser.add_argument("--update-stats", action="store_true", help="Only update stats instead of parsing a kyua db")
parser.add_argument("--add-prefix", help="Add a prefix to all testsuites")
args = parser.parse_args()
Expand Down
43 changes: 29 additions & 14 deletions test-scripts/run_bodiagsuite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,11 @@ def handle_testcase(self, o: Path, tools: list):

signaled = os.WIFSIGNALED(exit_code)
exited = os.WIFEXITED(exit_code)
testcase.system_out = "WIFSIGNALED={} WIFEXITED={}, WTERMSIG={}, WEXITSTATUS={}" \
" WCOREDUMP={}".format(signaled, exited, os.WTERMSIG(exit_code),
os.WEXITSTATUS(exit_code), os.WCOREDUMP(exit_code))
# -ok testcases are expected to run succesfully -> exit code zero
testcase.system_out = (
f"WIFSIGNALED={signaled} WIFEXITED={exited}, WTERMSIG={os.WTERMSIG(exit_code)}, "
f"WEXITSTATUS={os.WEXITSTATUS(exit_code)}, WCOREDUMP={os.WCOREDUMP(exit_code)}"
)
# -ok testcases are expected to run successfully -> exit code zero
if stem.endswith("-ok"):
if not exited or os.WEXITSTATUS(exit_code) != 0:
# This is not just a failure, it means something is seriously wrong if the good case fails
Expand All @@ -126,28 +127,34 @@ def handle_testcase(self, o: Path, tools: list):
testcase.result = junitparser.Error(message="INVALID OUTPUT FILE FOUND: " + o.name)
self.error_suite.add_testcase(testcase)
return
if exit_code == 1 and testcase.system_err and testcase.system_err.startswith(
"This test needs a CWD with length"):
if (
exit_code == 1
and testcase.system_err
and testcase.system_err.startswith("This test needs a CWD with length")
):
testcase.result = junitparser.Skipped(message="This test needs a large working directory")

# Handle tool-specific exit codes:
if "effectivesan" in tools:
# We do not instruct EffectiveSan to terminate on first error:
if "BOUNDS ERROR:\n" not in testcase.system_err:
testcase.result = junitparser.Failure(
message="EffectiveSan did not detect a bounds error. Exit code " + exit_code_str)
message="EffectiveSan did not detect a bounds error. Exit code " + exit_code_str,
)
elif "softboundcets" in tools:
# We do not instruct EffectiveSan to terminate on first error:
if "Softboundcets: Memory safety violation detected" not in testcase.system_err:
testcase.result = junitparser.Failure(
message="SoftBoundCETS did not detect a bounds error. Exit code " + exit_code_str)
message="SoftBoundCETS did not detect a bounds error. Exit code " + exit_code_str,
)
else:
# Otherwise we assume that the test must be killed by a signal
if not signaled:
# test should fail with a signal: (162 for CHERI)
# TODO: for CHERI check that it was signal 34?
testcase.result = junitparser.Failure(
message="Expected test to be killed by a SIGNAL but got exit code " + exit_code_str)
message="Expected test to be killed by a SIGNAL but got exit code " + exit_code_str,
)
suite.add_testcase(testcase)


Expand Down Expand Up @@ -208,8 +215,11 @@ def run_bodiagsuite(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespa
qemu.checked_run(f"cd {LONG_NAME_FOR_BUILDDIR} && mkdir -p run")
# Don't log all the CHERI traps while running (should speed up the tests a bit and produce shorter logfiles)
qemu.run("sysctl machdep.log_user_cheri_exceptions=0 || true")
qemu.checked_run(f"{args.bmake_path} -r -f {LONG_NAME_FOR_BUILDDIR}/Makefile.bsd-run all",
timeout=120 * 60, ignore_cheri_trap=True)
qemu.checked_run(
f"{args.bmake_path} -r -f {LONG_NAME_FOR_BUILDDIR}/Makefile.bsd-run all",
timeout=120 * 60,
ignore_cheri_trap=True,
)
# restore old behaviour
qemu.run("sysctl machdep.log_user_cheri_exceptions=1 || true")

Expand Down Expand Up @@ -246,9 +256,14 @@ def main():
sys.exit()

# we don't need ssh running to execute the tests
run_tests_main(test_function=run_bodiagsuite, need_ssh=False, should_mount_builddir=True,
argparse_setup_callback=add_args, build_dir_in_target=LONG_NAME_FOR_BUILDDIR)
run_tests_main(
test_function=run_bodiagsuite,
need_ssh=False,
should_mount_builddir=True,
argparse_setup_callback=add_args,
build_dir_in_target=LONG_NAME_FOR_BUILDDIR,
)


if __name__ == '__main__':
if __name__ == "__main__":
main()
119 changes: 84 additions & 35 deletions test-scripts/run_cheribsd_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,13 @@
from run_tests_common import CrossCompileTarget, boot_cheribsd, pexpect, run_tests_main


def run_cheribsdtest(qemu: boot_cheribsd.QemuCheriBSDInstance, binary_name, old_binary_names,
optional, args: argparse.Namespace) -> bool:
def run_cheribsdtest(
qemu: boot_cheribsd.QemuCheriBSDInstance,
binary_name,
old_binary_names,
optional,
args: argparse.Namespace,
) -> bool:
try:
qemu.checked_run(f"rm -f /tmp/{binary_name}.xml")
# Run it once with textual output (for debugging)
Expand Down Expand Up @@ -127,17 +132,18 @@ def run_cheribsd_test(qemu: boot_cheribsd.QemuCheriBSDInstance, args: argparse.N
# Disable trap dumps while running cheribsdtest (handle both old and new sysctl names until dev is merged):
qemu.run("sysctl machdep.log_user_cheri_exceptions=0 || sysctl machdep.log_cheri_exceptions=0")
cheribsdtest_bases = [
("cheribsdtest-hybrid", False),
("cheribsdtest-purecap", False),
("cheribsdtest-purecap-benchmark", True),
]
("cheribsdtest-hybrid", False),
("cheribsdtest-purecap", False),
("cheribsdtest-purecap-benchmark", True),
]
# The minimal disk image only has the statically linked base variants:
cheribsdtest_features = ["-dynamic", "-mt"] if not args.minimal_image else []
cheribsdtest_features_powerset = \
itertools.chain(*map(lambda r: itertools.combinations(cheribsdtest_features, r),
range(0, len(cheribsdtest_features)+1)))
cheribsdtest_tests = \
[(b[0] + ''.join(f), b[1]) for f in cheribsdtest_features_powerset for b in cheribsdtest_bases]
cheribsdtest_features_powerset = itertools.chain(
*map(lambda r: itertools.combinations(cheribsdtest_features, r), range(0, len(cheribsdtest_features) + 1)),
)
cheribsdtest_tests = [
(b[0] + "".join(f), b[1]) for f in cheribsdtest_features_powerset for b in cheribsdtest_bases
]
cheribsdtest_tests.append(("cheribsdtest-mt-c18n", True))
for test in cheribsdtest_tests:
if not run_cheribsdtest(qemu, test[0], [], test[1], args):
Expand All @@ -161,8 +167,12 @@ def run_cheribsd_test(qemu: boot_cheribsd.QemuCheriBSDInstance, args: argparse.N
test_start = datetime.datetime.now()
# Check that the file exists
qemu.checked_run(f"test -f {shlex.quote(tests_file)}")
qemu.run(f"kyua test --results-file=/tmp/results.db -k {shlex.quote(tests_file)}",
ignore_cheri_trap=True, cheri_trap_fatal=False, timeout=24 * 60 * 60)
qemu.run(
f"kyua test --results-file=/tmp/results.db -k {shlex.quote(tests_file)}",
ignore_cheri_trap=True,
cheri_trap_fatal=False,
timeout=24 * 60 * 60,
)
if i == 0:
result_name = "test-results.db"
else:
Expand Down Expand Up @@ -191,16 +201,22 @@ def run_cheribsd_test(qemu: boot_cheribsd.QemuCheriBSDInstance, args: argparse.N
boot_cheribsd.info("KYUA installed on the host, no need to do slow conversion in QEMU")
else:
xml_conversion_start = datetime.datetime.now()
qemu.checked_run("kyua report-junit --results-file=/tmp/results.db > /tmp/results.xml",
timeout=200 * 60)
qemu.checked_run(
"kyua report-junit --results-file=/tmp/results.db > /tmp/results.xml",
timeout=200 * 60,
)
if qemu.smb_failed:
boot_cheribsd.info("SMB mount has failed, performing normal scp")
qemu.scp_from_guest("/tmp/results.xml", Path(args.test_output_dir, results_xml.name))
else:
qemu.checked_run(f"cp -v /tmp/results.xml {results_xml}")
qemu.checked_run("fsync " + str(results_xml))
boot_cheribsd.success("Creating JUnit XML ", results_xml, " took: ",
datetime.datetime.now() - xml_conversion_start)
boot_cheribsd.success(
"Creating JUnit XML ",
results_xml,
" took: ",
datetime.datetime.now() - xml_conversion_start,
)
except boot_cheribsd.CheriBSDCommandTimeout as e:
boot_cheribsd.failure("Timeout running tests: " + str(e), exit=False)
qemu.sendintr()
Expand Down Expand Up @@ -290,28 +306,61 @@ def cheribsd_setup_args(args: argparse.Namespace):
if not boot_cheribsd.PRETEND:
(real_output_dir / "cmdline").write_text(str(sys.argv))
args.smb_mount_directories.append(
boot_cheribsd.SmbMount(real_output_dir, readonly=False, in_target="/test-results"))
boot_cheribsd.SmbMount(real_output_dir, readonly=False, in_target="/test-results"),
)


def add_args(parser: argparse.ArgumentParser):
parser.add_argument("--skip-poweroff", action="store_true",
help="Don't run poweroff after tests (implicit with --interact). Without --interact this will"
"almost certainly corrupt the disk image, so only pass this if you no longer need the "
"image!")
parser.add_argument("--kyua-tests-files", action="append", nargs=argparse.ZERO_OR_MORE, default=[],
help="Run tests for the given following Kyuafile(s)")
parser.add_argument(
"--skip-poweroff",
action="store_true",
help=(
"Don't run poweroff after tests (implicit with --interact). Without --interact this will"
"almost certainly corrupt the disk image, so only pass this if you no longer need the "
"image!"
),
)
parser.add_argument(
"--kyua-tests-files",
action="append",
nargs=argparse.ZERO_OR_MORE,
default=[],
help="Run tests for the given following Kyuafile(s)",
)
default_test_output = str(Path(".").resolve() / "cheribsd-test-results")
parser.add_argument("--test-output-dir", "--kyua-tests-output", dest="test_output_dir", default=default_test_output,
help="Directory for the test outputs (it will be mounted with SMB)")
parser.add_argument("--no-timestamped-test-subdir", action="store_true",
help="Don't create a timestamped subdirectory in the test output dir ")
parser.add_argument("--run-cheribsdtest", dest="run_cheribsdtest", action="store_true", default=None,
help="Run cheribsdtest programs")
parser.add_argument("--no-run-cheribsdtest", dest="run_cheribsdtest", action="store_false",
help="Do not run cheribsdtest programs")
parser.add_argument(
"--test-output-dir",
"--kyua-tests-output",
dest="test_output_dir",
default=default_test_output,
help="Directory for the test outputs (it will be mounted with SMB)",
)
parser.add_argument(
"--no-timestamped-test-subdir",
action="store_true",
help="Don't create a timestamped subdirectory in the test output dir ",
)
parser.add_argument(
"--run-cheribsdtest",
dest="run_cheribsdtest",
action="store_true",
default=None,
help="Run cheribsdtest programs",
)
parser.add_argument(
"--no-run-cheribsdtest",
dest="run_cheribsdtest",
action="store_false",
help="Do not run cheribsdtest programs",
)


if __name__ == '__main__':
if __name__ == "__main__":
# we set need_ssh to True here to test that SSH connections work.
run_tests_main(test_function=run_cheribsd_test, argparse_setup_callback=add_args, should_mount_builddir=False,
argparse_adjust_args_callback=cheribsd_setup_args, need_ssh=True)
run_tests_main(
test_function=run_cheribsd_test,
argparse_setup_callback=add_args,
should_mount_builddir=False,
argparse_adjust_args_callback=cheribsd_setup_args,
need_ssh=True,
)
47 changes: 33 additions & 14 deletions test-scripts/run_ctest_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def test_setup(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespace):
with cmake_cache.open("rb") as f:
for line in f.readlines():
if line.startswith(b"CMAKE_COMMAND:INTERNAL="):
host_cmake_path = line[len(b"CMAKE_COMMAND:INTERNAL="):].strip()
host_cmake_path = line[len(b"CMAKE_COMMAND:INTERNAL=") :].strip()
boot_cheribsd.info("Host CMake path is ", host_cmake_path)
break
for ctest_file in Path(args.build_dir).rglob("CTestTestfile.cmake"):
Expand Down Expand Up @@ -82,9 +82,12 @@ def run_ctest_tests(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespa
# First list all tests and then try running them.
qemu.checked_run(f"cd {args.build_dir} && /cmake/bin/ctest --show-only -V", timeout=5 * 60)
try:
qemu.checked_run(f"cd {args.build_dir} && /cmake/bin/ctest {ctest_args}",
timeout=int(args.test_timeout * 1.05), pretend_result=0,
ignore_cheri_trap=args.ignore_cheri_trap)
qemu.checked_run(
f"cd {args.build_dir} && /cmake/bin/ctest {ctest_args}",
timeout=int(args.test_timeout * 1.05),
pretend_result=0,
ignore_cheri_trap=args.ignore_cheri_trap,
)
except boot_cheribsd.CheriBSDCommandFailed as e:
boot_cheribsd.failure("Failed to run some tests: " + str(e), exit=False)
return False
Expand All @@ -95,20 +98,36 @@ def add_args(parser: argparse.ArgumentParser):
parser.add_argument("--cmake-install-dir", help="Installation root for the CMake/CTest commands", required=True)
parser.add_argument("--junit-xml", required=False, help="Output file name for the JUnit XML results")
parser.add_argument("--verbose", action="store_true", help="Enable verbose ctest output")
parser.add_argument("--ignore-cheri-trap", action="store_true", required=False, default=True,
help="Don't fail the tests when a CHERI trap happens")
parser.add_argument("--test-setup-command", action="append", dest="test_setup_commands", metavar="COMMAND",
help="Run COMMAND as an additional test setup step before running the tests")
parser.add_argument(
"--ignore-cheri-trap",
action="store_true",
required=False,
default=True,
help="Don't fail the tests when a CHERI trap happens",
)
parser.add_argument(
"--test-setup-command",
action="append",
dest="test_setup_commands",
metavar="COMMAND",
help="Run COMMAND as an additional test setup step before running the tests",
)


def adjust_args(args: argparse.Namespace):
args.smb_mount_directories.append(
boot_cheribsd.SmbMount(args.cmake_install_dir, readonly=True, in_target="/cmake"))
args.smb_mount_directories.append(boot_cheribsd.SmbMount(args.cmake_install_dir, readonly=True, in_target="/cmake"))
args.junit_xml = get_default_junit_xml_name(args.junit_xml, args.build_dir)


if __name__ == '__main__':
if __name__ == "__main__":
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_ctest_tests, test_setup_function=test_setup,
need_ssh=False, argparse_setup_callback=add_args, argparse_adjust_args_callback=adjust_args,
should_mount_builddir=True, should_mount_srcdir=True, should_mount_sysroot=True)
run_tests_main(
test_function=run_ctest_tests,
test_setup_function=test_setup,
need_ssh=False,
argparse_setup_callback=add_args,
argparse_adjust_args_callback=adjust_args,
should_mount_builddir=True,
should_mount_srcdir=True,
should_mount_sysroot=True,
)
Loading

0 comments on commit e45fa1c

Please sign in to comment.