diff --git a/bin/run_tests.py b/bin/run_tests.py new file mode 100755 index 000000000..a808d2514 --- /dev/null +++ b/bin/run_tests.py @@ -0,0 +1,551 @@ +#! /usr/bin/env python3 +# vim: set tabstop=4 shiftwidth=4 expandtab set textwidth=79 +# ===========================================================================79 +# Filename: run_tests.py +# +# Description: Converting run_tests.sh to python +# +# Based on run_test.sh +# +# Same as run_test.sh: +# 1. output into .xml file is the same +# 2. colorization of output +# +# Differences with run_test.sh +# 1. Does not depend upon .elf file extension for filetype. +# Uses the unix 'file' command to get filetype +# 2. Added usage function +# 3. Added run switches (which didn't exist in run_test.sh) +# See the print_usage() function for a descriptionm of +# the supported switches. +# 4. Default search directories, rather than just one. +# 5. Removed use of $RISCV env var. +# 6. requires python3 +# +# TODO: Need to add timeouts to tests. +# +# Author(s): Bill McSpadden (bill@riscv.org) +# +# History: See revision control log +# ===========================================================================79 + +import os # needed for os command interactions +import glob # needed for file list gathering useing wildcards +import re # regular expression +import sys # needed for command line arguments +import getopt # needed for command line arguments +import collections # needed for dequeues +import subprocess # needed for subprocesses where stdout is needed +from pathlib import Path +from inspect import currentframe, getframeinfo +from abc import ABC, abstractmethod +from copy import deepcopy + +print("starting...") +print("abspath to this script: " + os.path.abspath(sys.argv[0])) + +# Data structure for sim command line arguments +# TODO: make the key a regex +sim_test_command_line_switch = { + # test name sim command line switch + "rv32ui-v-ma_data" : "--enable-misaligned", + "rv32ui-p-ma_data" : "--enable-misaligned", + "rv64ui-v-ma_data" : "--enable-misaligned", + "rv64ui-p-ma_data" : "--enable-misaligned", + } + + + + +#set -e + +def TRACE(text = "") : + cf = currentframe() + of = cf.f_back + fi = getframeinfo(of) + filename = fi.filename + print("TRACE: file: " + filename + " line: " + str(of.f_lineno) + " : " + text) + return + +def print_usage(invocation) : + print(invocation + " usage: " + invocation + " []") + print(" Typically, invoke this script in the directory above where the elf-file tests live.") + print(" The script looks into test_dir, finds all of the elf files and then runs the simulator") + print(" with each elf file.") + print("") + print(" Output logs are put into [dir]/.cout (for C sim) or [dir/].out (for ocaml sim).") + print("") + print(" Some tests require specific command line switches to properly run. To add these") + print(" command line switches, you must edit this python script. Look for the array, 'sim_test_command_line_switch'") + print("") + print(" options:") + print(" -h --help -u -usage print out help/usage message") + print(" -o/--outfile= name of xml tests results file to be generated. default: ./tests.xml ") + print(" --32bit=[yes|y|no|n] run 32-bit tests. default: yes") + print(" --64bit=[yes|y|no|n] run 64-bit tests. default: yes") + print(" --c_sim=[yes|y|no|n] run the C simulator. default: yes") + print(" --ocaml_sim=[yes|y|no|n] run the Ocaml simulator. default: no") + print(" --clean_build=[yes|y|no|n] do a 'make clean' before running 32/64/c_sim/ocaml_sim set of tests. default: yes") + print(" --test_dir= directory where test elf files live. default: ./isa ./riscv-tests") + print(" Environment variable, 'ARCH', must be set to either 'RV32' or 'RV64'") + +opts, args = getopt.getopt ( + sys.argv[1:], + "huo:", + [ + "outfile=", + "test_dir=", + "32bit=", + "64bit=", + "c_sim=", + "ocaml_sim=", + "clean_build=", + "test_dir=" + ] + ) + +xml_outfile = "./tests.xml" +run_32bit_tests = True +run_64bit_tests = True +run_csim = True +run_ocamlsim = False +clean_build = True +test_dir_list = [ "isa", "riscv-tests" ] +sail_riscv_rootdir = 'SAIL_RISCV_ROOTDIR' + + +#print("opts: " + str(opts)) +for opt, arg in opts : + if opt in ('-h', 'help', '-u', '--usage') : + print_usage(sys.argv[0]) + sys.exit(0) + elif opt in ('-o', "--outfile") : + xml_outfile = arg + elif opt in ('--32bit') : + if arg in ('yes', 'y') : + run_32bit_tests= True + elif arg in ('no', 'n') : + run_32bit_tests= False + else : + print("error: invalid argument to '--32bit' switch: " + arg) + sys.exit(1) + elif opt in ('--64bit') : + if arg in ('yes', 'y') : + run_64bit_tests= True + elif arg in ('no', 'n') : + run_64bit_tests= False + else : + print("error: invalid argument to '--64bit' switch: " + arg) + sys.exit(1) + elif opt in ('--c_sim') : + if arg in ('yes', 'y') : + run_csim= True + elif arg in ('no', 'n') : + run_csim= False + else : + print("error: invalid argument to '--run_csim' switch: " + arg) + sys.exit(1) + elif opt in ('--ocaml_sim') : + if arg in ('yes', 'y') : + run_ocamlsim= True + elif arg in ('no', 'n') : + run_ocamlsim= False + else : + print("error: invalid argument to '--run_ocamlsim' switch: " + arg) + sys.exit(1) + elif opt in ('--clean_build') : + if arg in ('yes', 'y') : + clean_build= True + elif arg in ('no', 'n') : + clean_build= False + else : + print("error: invalid argument to '--run_ocamlsim' switch: " + arg) + sys.exit(1) + elif opt in ('--test_dir') : + if not os.path.exists(arg) : + print("error: test_dir path, '" + arg + "', does not exist") + sys.exit(1) + test_dir_list = [] + test_dir_list.append(arg) + else : + print("error: unexpected command line option: " + opt) + sys.exit(1) + +print('outfile: ', xml_outfile) +print('run_32bit_tests: ' + str(run_32bit_tests)) +print('run_64bit_tests: ' + str(run_64bit_tests)) +print('run_csim: ' + str(run_csim)) +print('run_ocamlsim: ' + str(run_ocamlsim)) +print('clean_build: ' + str(clean_build)) +print('test_dir_list: ' + str(test_dir_list)) +# TODO: check that only 1 dir in test_dir_list exists +for d in test_dir_list : + if os.path.exists(d) : + TESTDIR = d + else : + pass + +print('TESTDIR : ' + TESTDIR) + +# DIR points to the invocation directory. +DIR = os.getcwd() +SEARCH_DIR = DIR +while SEARCH_DIR != '/' : + if os.path.isfile(SEARCH_DIR + '/' + sail_riscv_rootdir) : + RISCVDIR = SEARCH_DIR + break + if SEARCH_DIR == '/' : + print("error: can't find root directory of repository") + sys.exit(1) + SEARCH_DIR = os.path.dirname(SEARCH_DIR) + +print("RISCVDIR: " + RISCVDIR) + +#RISCVDIR = DIR + '/..' + +# TODO: get rid of ARCH env var. It should not be +# used. +ARCH = os.environ.get("ARCH") + +if ARCH not in ('RV32', 'RV64') : + print_usage(sys.argv[0]) + sys.exit(1) + +RED = '\033[0;91m' +GREEN = '\033[0;92m' +YELLOW = '\033[0;93m' +NC = '\033[0m' + +if os.path.isfile(DIR + '/tests.xml') != False : + os.remove(DIR + '/tests.xml') + +test_pass = 0 +test_fail = 0 +all_pass = 0 +all_fail = 0 +SUITE_XML = "" +SUITES_XML = "" + +def green(test_str, ok_fail_str) : + global test_pass + global SUITE_XML + global GREEN + global NC + test_pass += 1 + print(test_str + ':' + GREEN + ok_fail_str + NC) + SUITE_XML += ' \n' + +def yellow(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global YELLOW + global NC + test_fail += 1 + print(test_str + ':' + YELLOW + ok_fail_str + NC) + SUITE_XML += ' \n ' + ok_fail_str + '\n \n' + +def red(test_str, ok_fail_str) : + global test_fail + global SUITE_XML + global RED + global NC + test_fail += 1 + print(test_str + ':' + RED + ok_fail_str + NC) + SUITE_XML += ' \n ' + ok_fail_str + '\n \n' + +def finish_suite(suite_name) : + global test_pass + global test_fail + global all_pass + global all_fail + global SUITE_XML + global SUITES_XML + + print(suite_name + ': Passed ' + str(test_pass) + ' out of ' + str(test_pass + test_fail) + '\n\n') + date_tmp = subprocess.check_output("date", shell=True, text=True) + date = date_tmp.rstrip() + SUITES_XML += ' \n' + SUITE_XML + ' \n' + SUITE_XML="" + all_pass += test_pass + all_fail += test_fail + test_pass = 0 + test_fail = 0 + +# TODO: there MUST be an equivalent to the 'file' command in python. +# Replace the 'file' command with a python equivalent. +def is_elf(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"ELF\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_32bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"32-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_64bit(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $1 } ' | grep -q \"64-bit\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv(filename) : + cmd = "file -b " + filename + " | awk 'BEGIN { FS = \",\" } ; { print $2 } ' | grep -q \"RISC-V\" " + ret = os.system(cmd) + if ret == 0 : + return 1 + else : + return 0 + +def is_riscv_elf(filename) : + return is_elf(filename) and is_riscv(filename) + +def is_riscv_elf_32(filename) : + return is_riscv_elf(filename) and is_32bit(filename) + +def is_riscv_elf_64(filename) : + return is_riscv_elf(filename) and is_64bit(filename) + +# TODO: check for success/failure of chdir +os.chdir(RISCVDIR) + +print("DIR + '/' + TESTDIR + '/' + * :", DIR + '/' + TESTDIR + '/' + "*") + +# Do 'make clean' to avoid cross-arch pollution. + +if clean_build : + cmd = "make clean" + ret_val = os.system(cmd) + if ret_val != 0 : + print("error: non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + +print("Building 32-bit RISCV specification...") +if run_ocamlsim : + if ARCH == 'RV32' : + cmd = "make ARCH=RV32 ocaml_emulator/riscv_ocaml_sim_RV32" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 32-bit RISCV OCaml emulator", "ok") + else : + print("error: non-zero exit value from command: '" + cmd + "'") + red("Building 32-bit RISCV OCaml emulator","fail") + +TRACE("run_32bit_tests and run_ocamlsim : " + str(run_32bit_tests and run_ocamlsim)) +if run_32bit_tests and run_ocamlsim : + TRACE() + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : +# print("test: " + test) + if not is_riscv_elf_32(test) : + continue + # skip F/D tests on OCaml for now + pat = re.compile('(rv32ud)') + mo = pat.search(test) + if mo != None : + continue + pat = re.compile('(rv32uf)') + mo = pat.search(test) + if mo != None : + continue + outfile = test + ".out" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + cmd = "timeout 5 " + RISCVDIR + "/ocaml_emulator/riscv_ocaml_sim_RV32" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("OCaml-32 " + os.path.basename(test), "ok") + else : + red("OCaml-32 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("32-bit RISCV OCaml-simulator tests") + +if clean_build : + cmd = "make clean" + ret_val = os.system(cmd) + if ret_val != 0 : + print("error: non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + + +print("Building 32-bit RISCV specification...") +if run_csim : + if ARCH == 'RV32' : + cmd = "make ARCH=RV32 SAILCOV=true c_emulator/riscv_sim_RV32" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 32-bit RISCV OCaml emulator", "ok") + else : + print("error: non-zero exit value from command: '" + cmd + "'") + red("Building 32-bit RISCV OCaml emulator","fail") + +TRACE("run_32bit_tests and run_csim : " + str(run_32bit_tests and run_csim)) +if run_32bit_tests and run_csim : + TRACE() + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : +# print("test: " + test) + if not is_riscv_elf_32(test) : + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV32" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + print("cmd: '" + cmd + "'") + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-32 " + os.path.basename(test), "ok") + else : + red("C-32 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("32-bit RISCV C-simulator tests") + +if clean_build : + cmd = "make clean" + ret_val = os.system(cmd) + if ret_val != 0 : + print("error: non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") +if run_ocamlsim : + ARCH = 'RV64' + if ARCH == 'RV64' : + cmd = "make ARCH=RV64 ocaml_emulator/riscv_ocaml_sim_RV64" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 64-bit RISCV OCaml emulator", "ok") + else : + print("error: non-zero exit value from command: '" + cmd + "'") + red("Building 64-bit RISCV OCaml emulator","fail") + +TRACE("run_64bit_tests and run_ocamlsim : " + str(run_64bit_tests and run_ocamlsim)) +if run_64bit_tests and run_ocamlsim : + TRACE() + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : +# print("test: " + test) + if not is_riscv_elf_64(test) : + continue + # skip F/D tests on OCaml for now + pat = re.compile('(rv64ud)') + mo = pat.search(test) + if mo != None : + continue + pat = re.compile('(rv64uf)') + mo = pat.search(test) + if mo != None : + continue + outfile = test + ".out" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + cmd = "timeout 5 " + RISCVDIR + "/ocaml_emulator/riscv_ocaml_sim_RV64" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("OCaml-64 " + os.path.basename(test), "ok") + else : + red("OCaml-64 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("64-bit RISCV OCaml-simulator tests") + + +if clean_build : + cmd = "make clean" + ret_val = os.system(cmd) + if ret_val != 0 : + print("error: non-zero exit value from command: '" + cmd + "'") + sys.exit(1) + else : + pass +else : + pass + +print("Building 64-bit RISCV specification...") +if run_csim : + ARCH = 'RV64' + if ARCH == 'RV64' : + cmd = "make ARCH=RV64 SAILCOV=true c_emulator/riscv_sim_RV64" + ret_val = os.system(cmd) + if ret_val == 0 : + green("Building 64-bit RISCV C emulator", "ok") + else : + print("error: non-zero exit value from command: '" + cmd + "'") + red("Building 64-bit RISCV C emulator","fail") + +TRACE("run_64bit_tests and run_csim : " + str(run_64bit_tests and run_csim)) +if run_64bit_tests and run_csim : + TRACE() + for test in glob.glob(DIR + '/' + TESTDIR + '/' + "*") : +# print("test: " + test) + if not is_riscv_elf_64(test) : + continue + outfile = test + ".cout" + sim_switch = "" + for key in sim_test_command_line_switch : + pat = re.compile(key) + mo = pat.search(test) + if mo != None: + sim_switch = sim_test_command_line_switch[key] + break + cmd = "timeout 5 " + RISCVDIR + "/c_emulator/riscv_sim_RV64" + " " + sim_switch + " " + test + " > " + outfile + " 2>&1 && grep -q SUCCESS " + outfile + ret_val = os.system(cmd) + if ret_val == 0 : + green("C-64 " + os.path.basename(test), "ok") + else : + red("C-64 " + os.path.basename(test), "fail") +else : + pass + +finish_suite("64-bit RISCV C-simulator tests") + +print('Passed ' + str(all_pass) + ' out of ' + str(all_pass + all_fail) + '\n\n') +XML = '\n' + SUITES_XML + '\n' + +xml_outfile_fh = open(DIR + '/' + xml_outfile, 'w') +print(XML, file = xml_outfile_fh) +xml_outfile_fh.close() + +if all_fail > 0 : + sys.exit(1) +else : + sys.exit(0) +