run-bootrom-tests

#! /usr/bin/env python

#
# Copyright (c) 2015 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from this
# software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

from __future__ import print_function
import os
import stat
import argparse
import shlex
import subprocess
from util import error, print_to_error


# Names of environment variables
LAST_SERVER_FFFF = ".LastServerFFFF"
LAST_BRIDGE_FFFF = ".LastBridgeFFFF"

# Environment variables
last_server_ffff = None
last_bridge_ffff = None

# Program return values
PROGRAM_SUCCESS = 0
PROGRAM_WARNINGS = 1
PROGRAM_ERRORS = 2


def ftdi_off():
    # Disable the normal ftdi-serial driver
    subprocess.check_call("modprobe -r -q ftdi_sio")
    subprocess.check_call("modprobe -r -q usbserial")


def ftdi_on():
    # (Re)enable the normal ftdi-serial driver
    subprocess.check_call("modprobe -q ftdi_sio")
    subprocess.check_call("modprobe -q usbserial")


def auto_int(x):
    # Workaround to allow hex numbers to be entered for numeric arguments
    return int(x, 16)


def getsetting(path, setting_name):
    # Get a setting from the specified setting file
    #
    # This is analogous to "getenv" and is intended as a workaround
    # for state varibles. When we're run as "sudo run-bootrom-tests"
    # we inherit root's environment, not the user's, so we store each
    # persistent state variable as a hidden file in the test folder.
    #
    # path The path to the test folder
    # setting_name The name of the setting file
    #
    # returns The value stored in the setting file, or None if the setting
    #         file doesn't exist
    setting = None
    pathname = os.path.join(path, setting_name)
    if (os.path.isfile(pathname)):
        with open(pathname, "r") as f:
            setting = f.readline()
    return setting


def putsetting(path, setting_name, setting):
    # Store a setting in the specified setting file
    #
    # This is analogous to "getenv" and is intended as a workaround
    # for state varibles. When we're run as "sudo run-bootrom-tests"
    # we inherit root's environment, not the user's, so we store each
    # persistent state variable as a hidden file in the test folder.
    #
    # path The path to the test folder
    # setting_name The name of the setting file
    # setting The value to set. (If None, then the setting file is removed.)
    #
    # returns Nothing
    pathname = os.path.join(path, setting_name)

    if not setting:
        if (os.path.isfile(pathname)):
            os.remove(pathname)
    else:
        with open(pathname, "w", 0666) as f:
            f.write(setting)
            os.chmod(pathname,
                     stat.S_IRUSR | stat.S_IWUSR |
                     stat.S_IROTH | stat.S_IWOTH |
                     stat.S_IRGRP | stat.S_IWGRP)


def validate_test_args(test_args):
    """Sanity-check the test args and return an error string

    Parameters:
        test_args The test script parser output

    Returns 'None' if the args are valid, or a string if invalid.
    """
    # The line is valid
    return None


def compare_log_to_response(log, resp):
    """ Search the log list for the responses in the response list

    Returns None if all the strings in the response list were found, in
    sequence, in the log. Otherwise, returns a string containing the
    response line number and text.
    """
    # Find our first non-blank line in the response list.
    response_line_no = 0
    while True:
        response_line = resp[response_line_no].rstrip()
        if len(response_line) > 0:
            break

        response_line_no += 1
        if response_line_no >= len(resp):
            # We ran through all of our respones lines, success!
            return "Blank response list"
        else:
            response_line = resp[response_line_no].rstrip()
            if len(response_line) > 0:
                break

    # Walk the log list, looking for matches against the response list
    for log_line in log:
        log_line = log_line.rstrip()
        if not log_line:
            # Blank line
            continue
        if response_line in log_line:
            # Found a match, step to the next non-blank line in the response
            # list
            while True:
                response_line_no += 1
                if response_line_no >= len(resp):
                    # We ran through all of our respones lines, success!
                    return None
                else:
                    response_line = resp[response_line_no].rstrip()
                    if len(response_line) > 0:
                        break
    return "{0:d}: {1:s}".format(response_line_no, response_line)


def process_response_file(log_file, response_file):
    """ Compare the test log_file against a response file

    Returns None if all the strings in the response file were found, in
    sequence, in the log_file. Otherwise, returns a string containing the
    response line number and text, or a string indicating the response file
    is missing (as appropriate).
    """
    # Read the log file into a list
    log = []
    with open(log_file, 'r') as f_desc:
        log = f_desc.readlines()

    # Read the response file into a list
    if os.path.isfile(response_file):
        response = []
        with open(response_file, 'r') as f_desc:
            response = f_desc.readlines()

        # Run the comparison and return the result
        return compare_log_to_response(log, response)
    else:
        print_to_error("Missing response file:", response_file)
        return "No response file"


def adapt_bin(bin):
    """ Ensure that the bootrom.bin files have a "-fpga" ornamentation """
    root, ext = os.path.splitext(bin)
    if root.endswith("-fpga"):
        # Already specified
        return bin
    else:
        # Add it in
        return root + "-fpga" + ext


def print_debug_log(log_file):
    # Print the captured debug log
    with open(log_file) as f:
        debug_output_list = f.readlines()

    print_to_error("Test log:")
    if not debug_output_list:
        print_to_error("    (No debug output)")
    else:
        print_to_error("\n".join(debug_output_list))
    print_to_error("")


def process_1_test(test_folder, bridge_bin, bridge_efuse, bridge_ffff,
                   server_bin, server_ffff, response_file, log_file,
                   timeout, ftdi_path, dummy_run):
    """Process a single test (batch analysis)

    From the parsed test_args, it will download the image, rboot the
    HAPS board, capture the output and check that against the response file.

    Parameters
        test_folder The path to the test suite folder (typically .../es3-test)
        bridge_bin The pathname of the bridge bootrom.bin image
        bridge_efuse The pathname of the bridge efuse file (xxx.efz)
        bridge_ffff The pathname of the bridge romimage file ("ffffxxx.bin)
        server_bin The pathname of the server bootrom.bin image
        server_ffff The pathname of the server efuse file (xxx.efz)
        response_file The pathname of the response file used to validate
            the test.
        log_file The path to the test test logs folder (typically
            .../es3-test/logs)
        timeout How many seconds of inactivity to wait before concluding the
            test has completed
        ftdi_path If None, just display the command that would be run.
            Otherwise, run the real test using <ftdi_path>/haps_test
        dummy_run If true, just display the command that would be run.
            If false, run the real test

    Returns A 2-element tuple consisting of:
        - Test-passed flag
        - Failed-reason string (contains line # in log file)
    """
    # Convert the args into a form that "haps_test" likes
    args = [os.path.join(ftdi_path, "haps_test")]
    args += ["--test_folder={0:s}".format(test_folder)]
    if bridge_bin:
        args += ["--bridge_bin={0:s}".format(bridge_bin)]
    if bridge_efuse:
        args += ["--efuse={0:s}".format(bridge_efuse)]
    if bridge_ffff:
        args += ["--bridge_ffff={0:s}".format(bridge_ffff)]
    if server_bin:
        args += ["--server_bin=={0:s}".format(server_bin)]
    if server_ffff:
        args += ["--server_ffff={0:s}".format(server_ffff)]
    args += ["--log={0:s}".format(log_file)]
    args += ["--timeout={0:d}".format(timeout)]

    # Run the test and capture the output
    if dummy_run:
        # Run a dummy test
        print("Would have run this command:\n", args)
        return (True, None)
    else:
        # Run the test and capture the output
        subprocess.check_call(args)

        # Check test results
        test_result = process_response_file(log_file, response_file)
        if test_result:
            # Test failed, "test_result" explains why
            return (False, test_result)
        else:
            # Test succeeded
            return (True, None)


def process_test_script(test_script, quick_test, verbose, timeout, ftdi_path,
                        dummy_run):
    """Process the test file (generated by create-bootrom-test-suite)

    Processes the test descriptor file, generating an output file
    in the folder referenced by test_path, and a set of modified
    BootRom.bin files.

    Parameters:
        test_script  The pathname of the test script file (xxx.ts)
        quick_test  If true, stop testing on the first failure. If false, run
            all of the tests
        verbose  (obvious)
        timeout How many seconds of inactivity to wait before concluding the
            test has completed
        ftdi_path If None, just display the command that would be run.
            Otherwise, run the real test using <ftdi_path>/haps_test
        dummy_run If true, just display the command that would be run.
            If false, run the real test

    Returns a 2-element tuple containing:
        - the number of tests that passed
        - the number of tests that failed
    """
    num_passed = 0
    num_failed = 0

    # Split the test_script into path and file_name, and assume the path
    # is the root of the test folder (e.g., "~/es3-test")
    # with the various bootrom.bin, FFFF files, response file, etc. in
    # standard subfolders.
    (path, script) = os.path.split(test_script)
    bootrom_path = os.path.join(path, "s1fw")
    efuse_path = os.path.join(path, "efuse")
    ffff_path = os.path.join(path, "ffff")
    response_path = os.path.join(path, "response-files")
    log_path = os.path.join(path, "logs")

    # Validate the paths
    if not os.path.isdir(path):
        raise ValueError("Missing test folder")
    if not os.path.isdir(bootrom_path):
        raise ValueError("Missing bootrom folder")
    if not os.path.isdir(efuse_path):
        raise ValueError("Missing efuse folder")
    if not os.path.isdir(ffff_path):
        raise ValueError("Missing ffff folder")
    if not os.path.isdir(response_path):
        raise ValueError("Missing response folder")

    # Import persistent settings into globals
    last_server_ffff = getsetting(path, LAST_SERVER_FFFF)
    last_bridge_ffff = getsetting(path, LAST_BRIDGE_FFFF)

    # Create the log folder if needed
    if not os.path.exists(log_path):
        os.makedirs(log_path)

    # Set up the test file parser
    parser = argparse.ArgumentParser(prog=script)
    parser.add_argument("--testname", "-t",
                        required=True,
                        help="The test name")

    parser.add_argument("--bin", "-b",
                        required=True,
                        help="The bridge BootRom file to load")

    parser.add_argument("--flash", "-f",
                        help="The bridge ffff file to flash")

    parser.add_argument("--efuse", "-e",
                        help="The pathname of the bridge e-Fuse file"
                        "(overrides default e-Fuse file)")

    parser.add_argument("--srvrbin", "-B",
                        help="The server BootRom file to load")

    parser.add_argument("--srvrflash", "-F",
                        help="The server ffff file to flash")

    parser.add_argument("--response", "-r",
                        required=True,
                        help="test-response (.rsp) file")

    # Now parse and process each line in the test file
    with open(test_script) as f_test:
        line_num = 1
        test_line = ""
        for line in f_test:
            # Handle continuation lines
            line = line.rstrip()
            if line[-1] == "\\":
                test_line += line[0:-1]
                # Note that, while we keep the line number correct, parsing
                # errors will be displaced to the last line number of a
                # continued line.
                line_num += 1
                continue
            else:
                test_line += line

            # Chop each line into a list, stripping comments and
            # preserving quoted strings
            test_descriptor = shlex.split(test_line, True)
            if test_descriptor:
                parser.prog = "{0:s} (line {1:d})".format(script, line_num)
                test_args = parser.parse_args(test_descriptor)
                error_string = validate_test_args(test_args)
                if error_string:
                    # Display the parsing error
                    raise ValueError("{0:s} {1:d}: {2:s}".
                                     format(script, line_num, error_string))
                else:
                    # Establish the default test settings
                    bridge_bin = None
                    bridge_ffff = None
                    bridge_efuse = None
                    server_bin = None
                    server_ffff = None
                    response_file = os.path.join(response_path,
                                                 test_args.testname + ".rsp")

                    # Pull in the test settings from the parser output
                    if test_args.bin:
                        bridge_bin = os.path.join(bootrom_path,
                                                  adapt_bin(test_args.bin))
                    if test_args.efuse:
                        bridge_efuse = os.path.join(efuse_path,
                                                    test_args.efuse)
                    if test_args.flash and \
                       (last_bridge_ffff != test_args.flash):
                        bridge_ffff = os.path.join(ffff_path,
                                                   test_args.flash)
                        last_bridge_ffff = test_args.flash
                        putsetting(path, LAST_BRIDGE_FFFF, last_bridge_ffff)
                    if test_args.srvrbin:
                        server_bin = os.path.join(bootrom_path,
                                                  adapt_bin(test_args.srvrbin))
                    if test_args.srvrflash and \
                       (last_server_ffff != test_args.srvrflash):
                        server_ffff = os.path.join(ffff_path,
                                                   test_args.srvrflash)
                        last_server_ffff = test_args.srvrflash
                        putsetting(path, LAST_SERVER_FFFF, last_server_ffff)
                    if test_args.response:
                        response_file = os.path.join(response_path,
                                                     test_args.response)
                    else:
                        response_file = os.path.join(response_path,
                                                     test_args.testname +
                                                     ".rsp")
                    log_file = os.path.join(log_path,
                                            test_args.testname + ".log")

                    # Run the test
                    test_passed, reason = process_1_test(path,
                                                         bridge_bin,
                                                         bridge_efuse,
                                                         bridge_ffff,
                                                         server_bin,
                                                         server_ffff,
                                                         response_file,
                                                         log_file,
                                                         timeout,
                                                         ftdi_path,
                                                         dummy_run)
                    if test_passed:
                        num_passed += 1
                        # Optionally display the test pass
                        if verbose:
                            print_to_error("Test '{0:s}' OK: {1:s}:".
                                           format(test_args.testname,
                                                  reason))
                            print_debug_log(log_file)
                    else:
                        num_failed += 1
                        # Display the test failure
                        error("Test '{0:s}' failed: {1:s}:".
                              format(test_args.testname, reason))
                        print_debug_log(log_file)

                        # In quick_test mode, stop the test suite on the
                        # first failure
                        if quick_test:
                            break
            line_num += 1
            test_line = ""
    return (num_passed, num_failed)


def main():
    """Run a test suite created from create-bootrom-tests"""
    parser = argparse.ArgumentParser()

    # Test args:
    parser.add_argument("--test", "-t",
                        required=True,
                        help="The pathname to the test script (.ts) file")

    parser.add_argument("--ftdi-path",
                        required=True,
                        help="The path to the 'haps_test' executable")

    parser.add_argument("--dummy",
                        action='store_true',
                        help="Validate the .tss file without runnng tests")

    parser.add_argument("--quick", "-q",
                        action='store_true',
                        help="Quick test: stop on first failure")

    parser.add_argument("--verbose", "-v",
                        action='store_true',
                        help="Quick test: stop on first failure")

    parser.add_argument("--timeout",
                        type=int,
                        default=5,
                        help="Debug serial timeout, in seconds")

    args = parser.parse_args()

    args.ftdi_path = os.path.expanduser(args.ftdi_path)

    # Run the test suite
    try:
        synopsis = process_test_script(args.test, args.quick, args.verbose,
                                       args.timeout, args.ftdi_path,
                                       args.dummy)
        print(synopsis[0], "passed", synopsis[1], "failed",
              synopsis[0] + synopsis[1], "total")
    except IOError as e:
        print_to_error("I/O Error: {0}".format(e))
    except ValueError as e:
        print_to_error("Value Error: {0}".format(e))
    except:
        error("Unknown error")
        raise


## Launch main
#
if __name__ == '__main__':
    main()