hsptest.hsp

#runtime "hsp3cl"
#packopt name "hsptest"
#packopt orgpath 1
#packopt version "VERSION"

#uselib "winmm.dll"
#cfunc timeGetTime "timeGetTime"
#uselib "shlwapi.dll"
#cfunc global PathIsRelativeA "PathIsRelativeA" sptr
#cfunc global PathRelativePathToA "PathRelativePathToA" sptr, sptr, sptr, sptr, sptr

#include "hspdef.as"
#include "hsptest_exception_code.as"
#include "hspext.as"
#include "hspcmp.as"
#include "kernel32.as"
#include "mod_regexp.as"
#include "hspinet.as"
#include "hsptest_result_format.as"
#include "argparse.hsp"
#include "make_testcall_script.hsp"
#include "console.hsp"
#include "logfile.hsp"
#include "parse_result.hsp"
#include "util.hsp"
#include "annotation.hsp"
#include "script_tool.hsp"

#define HSPTEST_TESTCALL_AX_NAME "__hsptest_start.ax"

version = "0.9"

sdim stdout, 2000
sdim ln, 2000

test_pass = 0
test_fail = 0
test_ignore = 0

// initialize console output
console_init

// parse commandline arguments
options = {"
-v --verbose 0
-s --stdout 0
-l --logfile 1
--openlog --openlog 0
-h --help 0
-r --runtime 1
--stop --stop 0
--version --version 0
"}
parse_args 32, options, dir_cmdline

// for debug
onerror goto *hsptest_error

// show usage
if is_exist_option("--help") {
    mes "usage: hsptest [options] [file_or_directory_path(s)]"
    mes ""
    mes "arguments:"
    mes "  file_or_directory_path(s)    the path(s) to the script file or directory containing the files you want to run the tests"
    mes ""
    mes "options:"
    mes "  -v, --verbose          print the result for each test"
    mes "  -s, --stdout           display captured stdout while test"
    mes "  -l, --logfile LOGPATH  write the log file to LOGPATH"
    mes "  -r, --runtime RUNTIME  specify runtime to run tests"
    mes "  --stop                 stop when exit hsptest"
    mes "  --openlog              open log file with hsed3"
    mes "  -h, --help             show this help message"
    mes "  --version              show hsptest version"
    end
}

// show version
if is_exist_option("--version") {
    mes "hsptest " + version
    end
}

// exit if runtime name is invalid
if is_exist_option("--runtime") && is_valid_runtime_name(get_option_arg("--runtime")) == 0 {
    color_mes "hsptest: error: " + get_option_arg("--runtime") + " is invalid runtime name", 0, FOREGROUND_RED
    end -1
}

// extract test scripts
file_num = 0
first_cwd = dir_cur
search_entry_num = get_arg_num()
if search_entry_num == 0 : search_entry_num = 1
repeat search_entry_num
    entry = get_arg(cnt)
    if entry == "" {
        if cnt == 0 {
            entry = "."
        } else {
            continue
        }
    }

    exist entry
    if strsize == -1 && exist_dir(entry) == 0 {
        color_mes "hsptest: error: directory " + entry + " is an invalid path", 0, FOREGROUND_RED
        end -1
    }

    if strsize != -1 {
        // �\�������t�@�C�����̐��`
        sdim rel_path, 260
        abs_pathes = get_absolute_path("."), get_absolute_path(entry)
        a = PathRelativePathToA(varptr(rel_path), abs_pathes(0), 0x10, abs_pathes(1), 0)
        strrep rel_path, "\\", "/" // �o�b�N�X���b�V�����p�X�̃f���~�^�ɂ���ƁA�e�X�g���[�`���Ăяo���X�N���v�g��include���ɓ˂����񂾂Ƃ��ɃG�X�P�[�v�V�[�P���X�ƌ�F����邽�߁A�f���~�^���X���b�V���ɒu������
        strrep rel_path, "./", ""
        notesel testcodes
        noteadd rel_path + ";" + getpath(entry, 32)
        file_num += 1
    } else {
        testcodes_in_dir = ""
        notesel testcodes_in_dir
        chdir entry
        dirlist testcodes_in_dir, "*.hsp", 1
        chdir first_cwd
        strrep testcodes_in_dir, "\n", ";" + entry + "\n"
        file_num += notemax + 1
        notesel testcodes
        noteadd testcodes_in_dir
    }
loop

// output first message
centered_mes " test session starts ", FOREGROUND_WHITE, "="
hspver_str = get_runtime_ver()
tmpstr = ""
repeat 4
    tmpstr += strmid(hspver_str, cnt, 1)
    if cnt == 3 : break
    tmpstr += "."
loop
hspver_str = tmpstr
mes "HSP " + hspver_str + ", hsptest " + version

label_pattern = "^(?:.*:)?[ \\t]*(\\*test\\w*)[^\\r]*$"

// count test num
all_test_num = 0
repeat file_num
    notesel testcodes
    noteget file_name_and_cwd, cnt
    if strtrim(file_name_and_cwd) == "" : continue
    split file_name_and_cwd, ";", file_name, code_cwd
    if strlen(match(file_name, "test")) == 0 : continue
    chdir first_cwd
    chdir code_cwd
    notesel testcode_buf
    noteload file_name
    remove_unused_data tmp, testcode_buf
    testcode_buf = tmp
    repeat
        matches label_match, testcode_buf, label_pattern, 0, 1, 0
        if stat == 0 : break
        notesel testcode_buf
        l = notefind(label_match(0), 0)
        notedel l
        all_test_num++
    loop
loop

sdim stdout_buf, 2000, all_test_num
sdim stdout_file_name, 256, all_test_num
sdim stdout_test_name, 64, all_test_num
sdim error_file_buf, 64, all_test_num
sdim error_detail_buf, 64, all_test_num
dim error_line_buf, all_test_num
dim stdout_result, all_test_num
sdim callstack_buf, 64, 1024, all_test_num

mes "collected " + all_test_num + " tests"
mes ""

test_num = 0

start_time = timeGetTime()

repeat file_num

    // continue if file is not test
    notesel testcodes
    noteget file_name_and_cwd, cnt
    split file_name_and_cwd, ";", file_name, code_cwd
    if strlen(match(file_name, "test")) == 0 : continue

    // change cwd to code directory
    chdir first_cwd
    chdir code_cwd

    notesel testcode_buf
    noteload file_name
    remove_unused_data tmp, testcode_buf
    testcode_buf = tmp

    if is_exist_option("--verbose") == 0 : mes file_name + " ", 1

    repeat

        // get test labels
        matches label_match, testcode_buf, label_pattern, 0, 1, 0
        if stat == 0 : break
        notesel testcode_buf
        l = notefind(label_match(0), 0)
        label_name = label_match(0, 1)
        test_name = strmid(label_name, 1, 1024)
        notedel l

        // get annotations
        annotation_init
        notesel testcode_buf
        if l - 1 >= 0 {
            noteget annotation_line, l - 1
            annotation_parse annotation_line
        }

        // select runtime
        runtime_name = get_annotation_attr_val("runtime")
        if runtime_name == "default" {
            if is_exist_option("--runtime") {
                runtime_name = get_option_arg("--runtime")
            } else {
                runtime_name = "hsp3cl"
            }
        }

        // continue next test if ignored
        if get_annotation_attr_val("ignore") == "true" || is_valid_runtime_name(runtime_name) == 0 {
            if is_valid_runtime_name(runtime_name) == 0 {
                color_mes "hsptest: error: " + runtime_name + " is invalid runtime name", 0, FOREGROUND_RED
            }
            stdout_file_name(test_num) = file_name
            stdout_test_name(test_num) = test_name
            if is_exist_option("--verbose") {
                mes file_name + "::" + test_name + ": ", 1
                color_mes "IGNORED", 1, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY
                right_justified_mes "[" + strf("%3d", int((double(test_num + 1) / double(all_test_num)) * 100.0)) + "%]", FOREGROUND_GREEN, 1
            } else {
                color_mes "-", 1, FOREGROUND_GREEN | FOREGROUND_BLUE
                right_justified_mes "[" + strf("%3d", int((double(test_num + 1) / double(all_test_num)) * 100.0)) + "%]", FOREGROUND_GREEN, 0
            }
            stdout_result(test_num) = 2
            test_num++
            test_ignore++
            continue
        }

        // make temporary testcall script
        temptest_path = make_temp_test(file_name, label_name, runtime_name)

        // compile test
        before_compipe_cwd = dir_cur
        chdir getpath(temptest_path, 32)
        hsc_ini temptest_path
        hsc_compath dir_exe + "\\common\\"
        hsc_comp 1, 0, 0
        hsc_getmes stdout
        hsc_bye
        chdir before_compipe_cwd
        if instr(stdout, 0, "#No error detected.") == -1 {
            strrep stdout, "\n", "\nhspcmp:"
            mes test_name + "@" + file_name + ": FAIL: compile error"
            mes "hspcmp:" + stdout
            test_fail++
            delete_temp_test
            continue
        }

        // copy executable ax file to test directory
        ax_file_path_in_temp_dir = getpath(temptest_path, 1) + ".ax"
        bcopy ax_file_path_in_temp_dir, HSPTEST_TESTCALL_AX_NAME

        // delete temp dir
        delete_temp_test

        // execute test
        dim process_info, 4
        dim sa_attr, 3
        runtime_stdout_handle_rd = 0 : runtime_stdout_handle_wr = 0
        sa_attr(0) = 12 : sa_attr(1) = 0 : sa_attr(2) = 1
        CreatePipe varptr(runtime_stdout_handle_rd), varptr(runtime_stdout_handle_wr), varptr(sa_attr), 0
        dim startup_info, 17
        startup_info(0) = 68 : startup_info(11) = 0x00000100 | 0x00000001 : startup_info(15) = runtime_stdout_handle_wr : startup_info(16) = runtime_stdout_handle_wr
        CreateProcess 0, dir_exe + "\\" + runtime_name + ".exe " + HSPTEST_TESTCALL_AX_NAME, 0, 0, 1, 0, 0, 0, varptr(startup_info), varptr(process_info)
        CloseHandle runtime_stdout_handle_wr
        if double(get_annotation_attr_val("timeout")) <= 0.0 {
            timeout = 0xFFFFFFFF
        } else {
            timeout = int(double(get_annotation_attr_val("timeout")) * 1000.0)
        }
        waitforsingleobject_status = WaitForSingleObject(process_info(0), timeout)
        if waitforsingleobject_status == 0x00000102 { // timeout
            stdout = get_fail_result_format(-1, "?", "Test timed out (" + get_annotation_attr_val("timeout") + " seconds)") + "\n"
            TerminateProcess process_info(0), -1
        } else {
            read_size = 0
            memset stdout, 0, varsize(stdout)
            sdim tmpbuf, 256
            repeat
                memset tmpbuf, 0, varsize(tmpbuf)
                if ReadFile(runtime_stdout_handle_rd, varptr(tmpbuf), varsize(tmpbuf), varptr(read_size), 0) == 0 : break
                stdout += tmpbuf
            loop
        }
        CloseHandle runtime_stdout_handle_rd

        // collect result
        sdim callstack, 64, 1024
        parse_result_format stdout, result, errline, errfile, errdetail, callstack, stdout
        stdout_buf(test_num) = stdout
        stdout_file_name(test_num) = file_name
        stdout_test_name(test_num) = test_name
        error_file_buf(test_num) = errfile
        error_line_buf(test_num) = int(errline)
        error_detail_buf(test_num) = errdetail
        repeat 1024
            if callstack(cnt) == "" : break
            callstack_buf(cnt, test_num) = callstack(cnt)
        loop

        // output result
        test_num++
        if is_exist_option("--verbose") {
            mes file_name + "::" + test_name + ": ", 1
            if result == "PASS" {
                color_mes "PASSED", 1, FOREGROUND_GREEN | FOREGROUND_INTENSITY
            } else {
                if errline == "-1" {
                    color_mes "FAILED: " + errdetail, 1, FOREGROUND_RED | FOREGROUND_INTENSITY
                } else {
                    color_mes "FAILED at line " + errline + " of " + errfile + ": " + errdetail, 1, FOREGROUND_RED | FOREGROUND_INTENSITY
                }
            }
            right_justified_mes "[" + strf("%3d", int((double(test_num) / double(all_test_num)) * 100.0)) + "%]", FOREGROUND_GREEN, 1
        } else {
            if result == "PASS" {
                color_mes ".", 1, FOREGROUND_GREEN
            } else {
                color_mes "F", 1, FOREGROUND_RED | FOREGROUND_INTENSITY
            }
            right_justified_mes "[" + strf("%3d", int((double(test_num) / double(all_test_num)) * 100.0)) + "%]", FOREGROUND_GREEN, 0
        }
        if result == "PASS" {
            test_pass++
        } else {
            test_fail++
            stdout_result(test_num - 1) = 1
        }

        // delete temporary testcall ax file
        delete HSPTEST_TESTCALL_AX_NAME
    loop
    if is_exist_option("--verbose") == 0 : mes ""
loop

if is_exist_option("--stdout") {
    stdout_exist = 0
    repeat all_test_num
        if stdout_buf(cnt) != "" {
            stdout_exist = 1
            centered_mes " Captured stdout call in " + stdout_file_name(cnt) + "::" + stdout_test_name(cnt) + " ", FOREGROUND_WHITE, "-"
            mes stdout_buf(cnt)
        }
    loop
    if stdout_exist {
        centered_mes "", FOREGROUND_WHITE, "-"
    } else {
        centered_mes " no stdout captured ", FOREGROUND_WHITE, "-"
    }
}

spend_time = strf("%.2f", double(timeGetTime() - start_time) / 1000.0)

mes ""
if is_exist_option("--logfile") || is_exist_option("--openlog") {
    init_logfile_module hspver_str, version, dir_cur
    repeat all_test_num
        append_result_log stdout_file_name(cnt), stdout_test_name(cnt), stdout_result(cnt)
        if stdout_buf(cnt) != "" : append_stdout_log stdout_file_name(cnt), stdout_test_name(cnt), stdout_buf(cnt)
        sdim callstack, 64, 1024
        ccnt = cnt
        repeat 1024
            if callstack_buf(cnt, ccnt) == "" : break
            callstack(cnt) = callstack_buf(cnt, ccnt)
        loop
        if stdout_result(cnt) == 1 : append_detail_log stdout_file_name(cnt), stdout_test_name(cnt), error_file_buf(cnt), error_line_buf(cnt), error_detail_buf(cnt), callstack
    loop
    chdir first_cwd
    if is_exist_option("--logfile") {
        logfile_path = get_option_arg("--logfile")
        make_logfile logfile_path
        mes "output the log file to " + dir_cur + "\\" + logfile_path
    }
    if is_exist_option("--logfile") && is_exist_option("--openlog") {
        exec "hsed3 " + dir_cur + "\\" + logfile_path
    } else : if is_exist_option("--logfile") == 0 && is_exist_option("--openlog") {
        getenv tempdir, "TEMP"
        if exist_dir(tempdir + "\\hsptest_log") == 0 {
            mkdir tempdir + "\\hsptest_log"
        }
        make_logfile tempdir + "\\hsptest_log\\log.log"
        exec "hsed3 " + tempdir + "\\hsptest_log\\log.log"
    }
}

if test_pass == 0 && test_fail == 0 {
    if test_ignore {
        centered_mes " no tests ran in " + spend_time + " seconds ( " + test_ignore + " tests ignored ) ", FOREGROUND_WHITE, "="
    } else {
        centered_mes " no tests ran in " + spend_time + " seconds ", FOREGROUND_WHITE, "="
    }
} else : if test_fail == 0 {
    if test_ignore {
        centered_mes " all " + str(test_pass) + " passed in " + spend_time + " seconds ( " + test_ignore + " tests ignored ) ", FOREGROUND_GREEN, "="
    } else {
        centered_mes " all " + str(test_pass) + " passed in " + spend_time + " seconds ", FOREGROUND_GREEN, "="
    }
} else {
    if test_ignore {
        centered_mes " " + str(test_fail) + " failed, " + str(test_pass) + " passed, " + test_ignore + " ignored in " + spend_time + " seconds ", FOREGROUND_RED, "="
    } else {
        centered_mes " " + str(test_fail) + " failed, " + str(test_pass) + " passed in " + spend_time + " seconds ", FOREGROUND_RED, "="
    }
}

if (is_exist_option("--stop")) {
    mes "Press Enter to stop..."
    input a, , 1
}
end test_fail

*hsptest_error
    mes "[internal error hsptest]"
    mes "hsptest_error(" + wparam + "): " + errmsg(wparam) + " in line " + lparam
    end wparam