bundleTest.sh

#!/bin/bash
#set -x
#echo "param:'$1'"

#
#------------------------------
#
# Import settings
#
# Git branch

. ./settings.sh

# WriteLog() function

. ./timestampLogger.sh


# UninstallHPCC() fuction

declare -f -F UninstallHPCC > /dev/null
    
if [ $? -ne 0 ]
then
    . ./UninstallHPCC.sh
fi

#
#------------------------------
#
# Function
#

usage()
{
    WriteLog "usage:" "/dev/null"
    WriteLog "  $0 [-b <bundle1_name>[,<bundle2_name>[,...]] [-noarchive] [-nostats] [-h]" "/dev/null"
    WriteLog "where:" "/dev/null"
    WriteLog " -b          - List of bundles to test. If missing all will be tested." "/dev/null"
    WriteLog " -noarchive  - Skip archive generation." "/dev/null"
    WriteLog " -nostats    - Skipp executing QueryStat2.py." "/dev/null"
    WriteLog " -h          - This help." "/dev/null"
    WriteLog " " "/dev/null"
}

ProcessLog()
{ 
    BUNDLE=$1
    TARGET=$2
    logfilename=$( ls -clr ${TEST_LOG_DIR}/$TARGET.*.log | head -1 | awk '{ print $9 }' )
    WriteLog "ML test result log filename: ${logfilename}" "${ML_TEST_LOG}"
    total=$(cat ${logfilename} | sed -n "s/^[[:space:]]*Queries:[[:space:]]*\([0-9]*\)[[:space:]]*$/\1/p")
    passed=$(cat ${logfilename} | sed -n "s/^[[:space:]]*Passing:[[:space:]]*\([0-9]*\)[[:space:]]*$/\1/p")
    failed=$(cat ${logfilename} | sed -n "s/^[[:space:]]*Failure:[[:space:]]*\([0-9]*\)[[:space:]]*$/\1/p")
    elapsed=$(cat ${logfilename} | sed -n "s/^Elapsed time: \(.*\)$/\1/p")

    WriteLog "TestResult:Total:${total} passed:${passed} failed:${failed} elaps:${elapsed}" "${ML_TEST_LOG}"
    echo "TestResult:$BUNDLE:total:${total} passed:${passed} failed:${failed} elaps:${elapsed}" >> ${ML_TEST_SUMMARY}
    
    # Rename result log file to name of the bundle
    logname=$(basename $logfilename)
    bundlelogfilename=${logname//$TARGET/$BUNDLE}
    printf "%s, %s\n" "$logname" "$bundlelogfilename"
    mv -v $logfilename ${TEST_LOG_DIR}/ml-$bundlelogfilename

}

#
#------------------------------
#
# Constants
#

LOG_DIR=~/HPCCSystems-regression/log

BIN_HOME=~/

#TEST_ROOT=${BUILD_DIR}/CE/platform
PLATFORM_HOME=${TEST_ROOT}
TEST_ENGINE_HOME=${PLATFORM_HOME}/testing/regress

LONG_DATE=$(date "+%Y-%m-%d_%H-%M-%S")
BUILD_LOG_FILE=${BIN_HOME}/"ML_build_"${LONG_DATE}".log";

BUNDLES_TO_TEST=( "ML_Core" "PBblas" "GLM" "DBSCAN" "GNN" "LearningTrees" "TextVectors" "KMeans" "SupportVectorMachines" "LinearRegression" "LogisticRegression" )
MAKE_ARCHIVE=1
GET_STAT=1

while [ $# -gt 0 ]
do
    param=$1
    param=${param//-/}
    upperParam=${param^^}
    WriteLog "Param: ${upperParam}" "/dev/null"
    case $upperParam in
        B*) shift
              BUNDLES_TO_TEST=( "$1" )
              ;;
              
        NOA*) MAKE_ARCHIVE=0
                ;;
                
        NOS*) GET_STAT=0
                ;;
    
        H* | *)
            WriteLog "Unknown parameter: ${upperParam}" "/dev/null"
            usage
            exit 1
            ;;
    esac
    shift
done

# For testing purposes
SKIP_INSTALL_BUNDLES=0

ML_CORE_VERSION="V3_0"
ML_PBLAS_VERSION="V3_0"
ML_TEST_ROOT=~/.HPCCSystems/bundles/_versions/PBblas/${ML_PBLAS_VERSION}/PBblas

ML_TEST_HOME=~/.HPCCSystems/bundles/_versions/
ML_TEST_LOG=${OBT_LOG_DIR}/ML_test-${LONG_DATE}.log
ML_CORE_REPO=https://github.com/hpcc-systems/ML_Core.git
ML_PBBLAST_REPO=https://github.com/hpcc-systems/PBblas.git

ML_TEST_RESULT_LOG=${OBT_LOG_DIR}/mltest.${LONG_DATE}.log
ML_TEST_SUMMARY=${OBT_LOG_DIR}/mltests.summary


TIMEOUTED_FILE_LISTPATH=${BIN_HOME}
TIMEOUTED_FILE_LIST_NAME=${TIMEOUTED_FILE_LISTPATH}/MlTimeoutedTests.csv
TIMEOUT_TAG="//timeout 900"

#
#----------------------------------------------------
#
# Start ML Test process
#

TIME_STAMP=$(date +%s)

WriteLog "ML test script log file: '${ML_TEST_LOG}'" "${ML_TEST_LOG}"
WriteLog "ML test result log file: '${ML_TEST_RESULT_LOG}'" "${ML_TEST_LOG}"

WriteLog "Machine Learning Confidence Test started" "${ML_TEST_LOG}"

WriteLog "path:'${PATH}'" "${ML_TEST_LOG}"
WriteLog "LD_LIBRARY_PATH:'${LD_LIBRARY_PATH}'" "${ML_TEST_LOG}"
WriteLog "GCC: $(gcc --version)" "${ML_TEST_LOG}"
WriteLog "CMake: $( /usr/local/bin/cmake --version | head -n 1 )" "${ML_TEST_LOG}"
WriteLog "Python: $(python --version  2>&1 )" "${ML_TEST_LOG}"
WriteLog "Python3: $(python3 --version  2>&1 )" "${ML_TEST_LOG}"
WriteLog "Tensorflow: $(pip3 list | egrep 'tensorflow ')" "${ML_TEST_LOG}"

WriteLog "System id: ${SYSTEM_ID}, HPCC_SERVICE = '${HPCC_SERVICE}'" "${ML_TEST_LOG}"

STARTUP_MSG=""

if [[ "$ML_BUILD" -eq 1 ]]
then
    STARTUP_MSG=${STARTUP_MSG}"build"
fi

if [[ "$ML_RUN_THOR" -eq 1 ]]
then
    [[ -n "$STARTUP_MSG" ]] && STARTUP_MSG+=", "
    STARTUP_MSG="${STARTUP_MSG}thor"
fi

if [[ -n "$STARTUP_MSG" ]]
then
    WriteLog "Execute bundle tests on ${STARTUP_MSG}" "${ML_TEST_LOG}"
else
    WriteLog "No target selected. This is a dry run." "${ML_TEST_LOG}"
fi

WriteLog "Bundles to tests ${BUNDLES_TO_TEST[*]}" "${ML_TEST_LOG}"

#
#---------------------------
#
# Determine the package manager

WriteLog "packageExt: '${PKG_EXT}', installCMD: '${PKG_INST_CMD}'." "${ML_TEST_LOG}"

#
#---------------------------
#
# Clean system
#

WriteLog "Clean system" "${ML_TEST_LOG}"

[ ! -e $ML_TEST_ROOT ] && mkdir -p $ML_TEST_ROOT

#rm -rf ${PERF_TEST_ROOT}/*
cd  ${ML_TEST_ROOT}

 
#
#---------------------------
#
# Uninstall HPCC to free as much disk space as can
#

if [[ ${ML_KEEP_HPCC} -eq 0 ]]
then
    WriteLog "Uninstall HPCC to free as much disk space as can" "${ML_TEST_LOG}"
    
    WriteLog "Uninstall HPCC-Platform" "${ML_TEST_LOG}"
    
    UninstallHPCC "${ML_TEST_LOG}" "${ML_WIPE_OFF_HPCC}"
else
    WriteLog "Skip Uninstall HPCC but stop it!" "${ML_TEST_LOG}"
    StopHpcc "${ML_TEST_LOG}"
fi

#
#--------------------------------------------------
#
# Build it
#

if [[ $ML_BUILD -eq 1 ]]
then
    WriteLog "                                           " "${ML_TEST_LOG}"
    WriteLog "*******************************************" "${ML_TEST_LOG}"
    WriteLog " Build HPCC Platform from ${BUILD_HOME} ..." "${ML_TEST_LOG}"
    WriteLog "                                           " "${ML_TEST_LOG}"

    #
    #-------------------------------------------------------------------------------------
    # Build HPCC
    #
    
    cd ${BUILD_HOME}

    date=$( date "+%Y-%m-%d %H:%M:%S")
    WriteLog "Start build at ${date}" "${ML_TEST_LOG}"

    if [ ! -f  ${BUILD_DIR}/bin/build_pf.sh ]
    then
        C_CMD="cmake -DUSE_LIBXSLT=ON -DXALAN_LIBRARIES= -D CMAKE_BUILD_TYPE=$ML_BUILD_TYPE -D CMAKE_ECLIPSE_MAKE_ARGUMENTS=-30 -DECLWATCH_BUILD_STRATEGY='IF_MISSING' ../HPCC-Platform ln -s ../HPCC-Platform"
        WriteLog "${C_CMD}" "${ML_TEST_LOG}"

        res=( "$(${C_CMD} 2>&1)" )
        echo "${res[*]}" > $BUILD_LOG_FILE
    else
        res=( ". $(${BUILD_DIR}/bin/build_pf.sh HPCC-Platform 2>&1)" )
        echo "${res[*]}"  > $BUILD_LOG_FILE

        WriteLog "Execute '${BUILD_DIR}/bin/build_pf.sh'" "${ML_TEST_LOG}"

    fi

    CMD="sudo make -j ${NUMBER_OF_CPUS} package"

    WriteLog "cmd: ${CMD}" "${ML_TEST_LOG}"

    ${CMD} >> ${BUILD_LOG_FILE} 2>&1

    #res=$( ${CMD} 2>&1 )
    #WriteLog "build result:${res}" "${ML_TEST_LOG}"

    if [ $? -ne 0 ] 
    then
        WriteLog "Build failed: build has errors " "${ML_TEST_LOG}"
        buildResult=FAILED
        exit
    else
        ls -l hpcc*${PKG_EXT} >/dev/null 2>&1
        if [ $? -ne 0 ] 
        then
            WriteLog "Build failed: no rpm package found " "${ML_TEST_LOG}"
            buildResult=FAILED
            exit
        else
            WriteLog "Build succeed" "${ML_TEST_LOG}"
            buildResult=SUCCEED
        fi
    fi

    date=$( date "+%Y-%m-%d %H:%M:%S")
    WriteLog "Build end at ${date}" "${ML_TEST_LOG}"

    HPCC_PACKAGE=$( grep 'Current release version' ${BUILD_LOG_FILE} | cut -c 31- )${PKG_EXT}
    
    WriteLog " Default package: '${HPCC_PACKAGE}'." "${ML_TEST_LOG}"
    WriteLog "                                           " "${ML_TEST_LOG}"      

else
    WriteLog "                                           " "${ML_TEST_LOG}"
    WriteLog "*******************************************" "${ML_TEST_LOG}"
    WriteLog " Skip build HPCC Platform...               " "${ML_TEST_LOG}"
    WriteLog "                                           " "${ML_TEST_LOG}"      

    cd ${BUILD_HOME}
    #HPCC_PACKAGE=$(find . -maxdepth 1 -name 'hpccsystems-platform-community*' -type f | sort -r | head -n 1 )    
    HPCC_PACKAGE=$(find ~/build/CE/platform/build/ -maxdepth 1 -iname 'hpccsystems-platform*' -type f | sort -rV | head -n 1 )
    WriteLog " Default package: \n\t'${HPCC_PACKAGE}' ." "${ML_TEST_LOG}"
    WriteLog "                                           " "${ML_TEST_LOG}"      

fi

TARGET_PLATFORM="thor"

if [ $ML_RUN_THOR -eq 1 ]
then
    #***************************************************************
    #
    #           THOR test
    #
    #***************************************************************
    
    WriteLog "                                   " "${ML_TEST_LOG}"
    WriteLog "***********************************" "${ML_TEST_LOG}"
    WriteLog " Start ML ${TARGET_PLATFORM} test. " "${ML_TEST_LOG}"
    WriteLog "                                   " "${ML_TEST_LOG}"
    
    #
    # --------------------------------------------------------------
    # Install HPCC
    #
    if [[ ${ML_KEEP_HPCC} -eq 0 ]]
    then
        WriteLog "Install HPCC Platform ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        
        res=$( ${SUDO} ${PKG_INST_CMD} ${HPCC_PACKAGE} 2>&1 )
        retCode=$?
        WriteLog "ret code:$retCode" "${ML_TEST_LOG}"
        
        if [ $retCode -ne 0 ]
        then
            if [[ "$res" =~ "already installed" ]]
            then
                WriteLog "$res" "${ML_TEST_LOG}"
                WriteLog "REmove installed one..." "${ML_TEST_LOG}"
                res=$( ${SUDO} ${PKG_REM_CMD} hpccsystems-platform 2>&1 )
                retCode=$?
                WriteLog "ret code:$retCode" "${ML_TEST_LOG}"
                WriteLog "res:$res" "${ML_TEST_LOG}"
                WriteLog "$( sudo rm -v /etc/HPCCSystems/environment.xml)\n" "${ML_TEST_LOG}"
                WriteLog "Install the latest '${PKG_INST_CMD} ${HPCC_PACKAGE}' ..." "${ML_TEST_LOG}"
                res=$( ${SUDO} ${PKG_INST_CMD} ${HPCC_PACKAGE} 2>&1 )
                retCode=$?
                WriteLog "ret code:$retCode" "${ML_TEST_LOG}"
                WriteLog "res:$res" "${ML_TEST_LOG}"
            else
                WriteLog "Error in install! ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
                exit
            fi
        fi
    else
        WriteLog "Use the already installed ${HPCC_PACKAGE} version." "${ML_TEST_LOG}"
    fi

    #
    #---------------------------
    #
    # Patch environment.xml to use diferent size Memory
    #

    MEMSIZE=$(( $ML_THOR_MEMSIZE_GB * (2 ** 30) ))
    MEMSIZE_KB=$(( $ML_THOR_MEMSIZE_GB * (2 ** 20) ))

    # for hthor we should change 'defaultMemoryLimitMB' as well 
    MEMSIZE_MB=$(( $ML_THOR_MEMSIZE_GB * (2 ** 10) ))
    WriteLog "Patch environment.xml to use ${ML_THOR_MEMSIZE_GB}GB Memory for test on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    WriteLog "Patch environment.xml to use ${ML_THOR_NUMBER_OF_SLAVES} slaves for ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    
    WriteLog "$( ${SUDO} cp -v /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.xml.bak )"  "${ML_TEST_LOG}"
    ${SUDO} sed -e 's/totalMemoryLimit="1073741824"/totalMemoryLimit="'${MEMSIZE}'"/g' -e 's/slavesPerNode="1"/slavesPerNode="'${ML_THOR_NUMBER_OF_SLAVES}'"/g' "/etc/HPCCSystems/environment.xml" > temp.xml && ${SUDO} mv -f temp.xml "/etc/HPCCSystems/environment.xml"

    if [ $? -ne 0 ]
    then
        WriteLog "Error in update environment.xml file! ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    else
        WriteLog "The environment.xml file Updated. ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        WriteLog "$(ls -l /etc/HPCCSystems/environment.xml)" "${ML_TEST_LOG}"
        WriteLog "$(egrep 'totalMemoryLimit=|slavesPerNode=' /etc/HPCCSystems/environment.xml)" "${ML_TEST_LOG}"
    fi

    #
    #----------------------------------------------------
    #
    # Kill Cassandra if it used too much memory
    #
    WriteLog "Check memory on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    
    freeMem=$( GetFreeMem )

    WriteLog "Free memory is: $( GetFreeMemGB ) on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    
    if [[ $freeMem -lt $MEMSIZE_KB ]]
    then
        WriteLog "Free memory too low on ${TARGET_PLATFORM} and we need ${ML_THOR_MEMSIZE_GB}.!" "${ML_TEST_LOG}"

        cassandraPID=$( ps ax  | grep '[c]assandra' | awk '{print $1}' )
    
        if [ -n "$cassandraPID" ]
        then
            WriteLog "Kill Cassandra (pid: ${cassandraPID})  on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        
            ${SUDO}  kill -9 ${cassandraPID}
            sleep 5
    
            freeMem=$( GetFreeMem  )
            if [[ "$freeMem" -lt 3777356 ]]
            then
                WriteLog "The free memory ($( GetFreeMemGB )) is still too low! Cannot start HPCC Systems!! Give it up on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        
                # send email to Agyi
                echo "After the kill Cassandra the Performance test free memory (${freeMem} kB) is still too low on ${TARGET_PLATFORM}! Performance test stopped!" | mailx -s "OBT Memory problem" -u $USER  ${ADMIN_EMAIL_ADDRESS}
        
                exit -1
            fi
        fi
        WriteLog "Free memory is: $( GetFreeMemGB ) on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
    fi

    #
    #---------------------------
    #
    # Check HPCC Systems
    #
    WriteLog "Check HPCC Systems on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"

    if [[ ! -f /opt/HPCCSystems/sbin/configgen ]]
    then
        WriteLog "I can't find the HPCC Platform. Please install it and try again." "${ML_TEST_LOG}"
        exit -3
    fi
    
    NUMBER_OF_HPCC_COMPONENTS=$( /opt/HPCCSystems/sbin/configgen -env /etc/HPCCSystems/environment.xml -list | egrep -i -v 'eclagent' | wc -l )
    
    if [[ $NUMBER_OF_HPCC_COMPONENTS -eq 0 ]]
    then
        WriteLog "Unable start HPCC system!! Only ${NUMBER_OF_HPCC_COMPONENTS} component is configured to run." "${ML_TEST_LOG}"
        exit -2
    fi

    hpccRunning=$( ${HPCC_SERVICE} status | grep -c 'running' )

    WriteLog "Number of HPCC components:$NUMBER_OF_HPCC_COMPONENTS, running:$hpccRunning" "${ML_TEST_LOG}"

    if [[ "$hpccRunning" -ne "$NUMBER_OF_HPCC_COMPONENTS" ]]
    then
        WriteLog "Start HPCC System on ${TARGET_PLATFORM}..." "${ML_TEST_LOG}"
        
        hpccStatus=$( ${HPCC_SERVICE} start  2>&1 )
 
        WriteLog "Result:\n${hpccStatus}" "${ML_TEST_LOG}"
    fi
    
    # give it some time
    sleep 5
    
    hpccRunning=$( ${HPCC_SERVICE} status | grep -c 'running' )
    if [[ "$hpccRunning" -ne ${NUMBER_OF_HPCC_COMPONENTS} ]]
    then
        WriteLog "Unable start HPCC system!! Only ${hpccRunning} component is up on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        exit -2
    else
        hpccStatus=$( ${HPCC_SERVICE} status )
        WriteLog "HPCC system status: \n${hpccStatus}" "${ML_TEST_LOG}"
    fi
    
    #
    #---------------------------
    #
    # Get test from github
    #
    WriteLog "Get the latest ML bundles" "${ML_TEST_LOG}"
    
    cd  ${PERF_TEST_ROOT}
    myPwd=$( pwd )

    WriteLog "Pwd: '${myPwd}' for $TARGET_PLATFORM" "${ML_TEST_LOG}"

    BUNDLES_COUNT=${#BUNDLES_TO_TEST[@]}
    [[ $SKIP_INSTALL_BUNDLES -eq 1 ]] && BUNDLES_COUNT=0
    
    for ((i=0; i<$BUNDLES_COUNT; i++))
    do
        BUNDLE_NAME=${BUNDLES_TO_TEST[i]}
        WriteLog "Bundle: $BUNDLE_NAME" "${ML_TEST_LOG}"
        BUNDLE_REPO="https://github.com/hpcc-systems/${BUNDLES_TO_TEST[i]}.git"
        WriteLog "Repo: $BUNDLE_REPO" "${ML_TEST_LOG}"
        INSTALL_CMD="ecl bundle install --update --force $ML_INSTALL_EXTRA ${BUNDLE_REPO}"
        WriteLog "Install cmd: $INSTALL_CMD" "${ML_TEST_LOG}"
     
        tryCountMax=5
        tryCount=$tryCountMax
        tryDelay=1m

        while true
        do
            cRes=$( ${INSTALL_CMD} 2>&1 )
            retCode=$?
            if [[ 0 -ne  $retCode ]]
            then
                WriteLog "retCode: $retCode" "${ML_TEST_LOG}"
                WriteLog "cRes   : '$cRes'" "${ML_TEST_LOG}"
                tryCount=$(( $tryCount-1 ))

                if [[ $tryCount -ne 0 ]]
                then
                    WriteLog "Wait for ${tryDelay} to try again." "${ML_TEST_LOG}"
                    sleep ${tryDelay}
                    continue
                else
                    WriteLog "Install $BUNDLE_NAME bundle was failed after ${tryCountMax} attempts. Result is: '${cRes}'" "${ML_TEST_LOG}"
                    break;
                    # Don't give up, try the next bundle
                    #WriteLog "Archive ${TARGET_PLATFORM} ML logs" "${ML_TEST_LOG}"
                    #${BIN_HOME}/archiveLogs.sh ml-${TARGET_PLATFORM} timestamp=${OBT_TIMESTAMP}
                    #exit -3
                fi
            else
                WriteLog "Install $BUNDLE_NAMEbundle was success." "${ML_TEST_LOG}"
                BUNDLE_VERSION=$( echo "${cRes}" | egrep "^$BUNDLE_NAME" | awk '{ print $2 }' )
                WriteLog "Version: $BUNDLE_VERSION" "${ML_TEST_LOG}"
                break
            fi
        done
        
        WriteLog ".............................." "${ML_TEST_LOG}"

    done
    
    if [[ ! -d ${ML_TEST_HOME} ]]
    then
        WriteLog "${ML_TEST_HOME} doesn't exists! Check the version, maybe changed!" "${ML_TEST_LOG}"
        exit -4
    fi

    cd  ${ML_TEST_HOME}

#    echo "Copy new tests over...."
#    read
    myPwd=$( pwd )
    #
    #---------------------------
    #
    # Run ML tests 
    #
    WriteLog "Run ML tests  on platforms pwd:${myPwd}" "${ML_TEST_LOG}"
    
    WriteLog "ML_TEST_HOME          : ${ML_TEST_HOME}" "${ML_TEST_LOG}"
    WriteLog "ML_ENGINE_HOME        : ${ML_ENGINE_HOME}" "${ML_TEST_LOG}"
    WriteLog "Bundles to try to test: ${BUNDLES_TO_TEST[*]}" "${ML_TEST_LOG}"

    CMD="${REGRESSION_TEST_ENGINE_HOME}/ecl-test run -t ${TARGET_PLATFORM} --config ${REGRESSION_TEST_ENGINE_HOME}/ecl-test.json --timeout ${ML_TIMEOUT} $ML_REGRESSION_EXTRA_PARAM --pq ${ML_PARALLEL_QUERIES} ${ML_EXCLUDE_FILES}"

    if [ ${EXECUTE_ML_SUITE} -ne 0 ]
    then
        while read bundle
        do
            bundleRunPath=${bundle%/ecl}                         # remove '/ecl' from the end of the $bundle
            bundlePath=${bundleRunPath%/OBTTests};       # remove '/OBTTests' from the end of the $bundleRunPath if exists
            bundleName=${bundlePath%/test}                    # remove '/test' from the end of the $bundlePath if exists
            bundleName=$(basename $bundleName )         # remove path from $bundleName
            
            if [[ "$bundle" =~ "LearningTrees" ]]
            then
                # add a warning supression parameter in the file
                for file in "RegressionTestModified.ecl" "ClassificationTestModified.ecl"
                do
                    if [[ $( egrep -c '#ONWARNING\(30004' $bundle/$file ) -eq 0 ]]
                    then
                        WriteLog "Patch the '$file' to avoid execution time skew warning in $bundleName bundle." "${ML_TEST_LOG}"
                        pushd $bundle 
                        cp -fv $file $file-back
                        # Insert a comment and the "#ONWARNING" after the Copyright header
                        sed -i '/## \*\//a \\n// Patched by the bundleTest on '"$( date '+%Y.%m.%d %H:%M:%S')"' \n#ONWARNING(30004, ignore); // Do not report execute time skew warning' $file
                        WriteLog "Check the result:\n$( egrep '#ONWARNING\(' $file )\n" "${ML_TEST_LOG}"
                        popd
                    else
                        WriteLog "The '$file' is already fixed in $bundleName bundle." "${ML_TEST_LOG}"
                    fi
                done
            fi
            
            if [[ ! "${BUNDLES_TO_TEST[*]}" =~ "$bundleName"  ]]
            then
                WriteLog "Skip $bundleName test." "${ML_TEST_LOG}"
                continue
            fi
            
            WriteLog "Bundle with Regression Test: $bundleName" "${ML_TEST_LOG}"
            pushd $bundleRunPath
        
            WriteLog "CMD: '${CMD}'" "${ML_TEST_LOG}"
        
            ${CMD} >> ${ML_TEST_LOG} 2>&1
            
            retCode=$( echo $? )
            WriteLog "retcode: ${retCode}" "${ML_TEST_LOG}"
        
            if [ ${retCode} -ne 0 ] 
            then
                WriteLog "Machine Learning $bundle tests on ${TARGET_PLATFORM} returns with ${retCode}" "${ML_TEST_LOG}"
                #exit -1
            else 
                ProcessLog "$bundleName" "${TARGET_PLATFORM}"
            fi
           
            popd
            
            WriteLog ".............................." "${ML_TEST_LOG}"
            
        done< <(find . -iname 'ecl' -type d | sort )
    else
                WriteLog "Skip Machine Learning test suite execution!" "${ML_TEST_LOG}"
                WriteLog "                                      " "${ML_TEST_LOG}"        
    fi

    # Generate ZAP file if specified in settings.sh
    if [  ${#ML_GENERATE_ZAP_FOR[@]} -ne 0 ]
    then
        for test in ${ML_GENERATE_ZAP_FOR[*]}
        do 
            WriteLog "Generate ZAP file for '$test'"  "${ML_TEST_LOG}"
            wuid=$(ecl getwuid -n $test --limit 1)
            if [ -n $wuid ]
            then
                WriteLog "  WUID: '$wuid'"  "${ML_TEST_LOG}"
                res=$(ecl zapgen $wuid  --path ${ZAP_DIR}/  --inc-thor-slave-logs)
                retCode=$?
                WriteLog "retCode: $retCode"  "${ML_TEST_LOG}"
                WriteLog "res    : $res"  "${ML_TEST_LOG}"
            else
                WriteLog "Workunit not found."  "${ML_TEST_LOG}"
            fi
        done
    fi
    
    
    NUM_OF_ML_ZAPS=( $(sudo find ${ZAP_DIR}/ -iname 'ZAPReport*' -type f -exec printf "%s\n" '{}' \; ) )
    if [ ${#NUM_OF_ML_ZAPS[@]} -ne 0 ]
    then
        WriteLog "Copy ML test ZAP files to ${TARGET_DIR}/test/ZAP" "${ML_TEST_LOG}"
        if [ ! -e ${TARGET_DIR}/test/ZAP ]
        then
            WriteLog "Create ${TARGET_DIR}/test/ZAP directory..." "${ML_TEST_LOG}"
            mkdir -p ${TARGET_DIR}/test/ZAP
        fi
    
        WriteLog "cp ${ZAP_DIR}/* ${TARGET_DIR}/test/ZAP/" "${ML_TEST_LOG}"
        cp ${ZAP_DIR}/* ${TARGET_DIR}/test/ZAP/
    else
        WriteLog "No ZAP file generated." "${ML_TEST_LOG}"
    fi

    # Check if any core file generated. If yes, create stack trace with gdb
    NUM_OF_ML_CORES=( $(sudo find /var/lib/HPCCSystems/ -iname 'core*' -mtime -1 -type f -exec printf "%s\n" '{}' \; ) )
    
    if [ ${#NUM_OF_ML_CORES[@]} -ne 0 ]
    then
        WriteLog "${#NUM_OF_ML_CORES[@]} ML test core files found." "${ML_TEST_LOG}"

        for  core in ${NUM_OF_ML_CORES[@]}
        do
            WriteLog "Generate backtrace for $core." "${ML_TEST_LOG}"
            base=$( dirname $core )
            lastSubdir=${base##*/}
            comp=${lastSubdir##my}
    
            #sudo gdb --batch --quiet -ex "set interactive-mode off" -ex "thread apply all bt" -ex "quit" "/opt/HPCCSystems/bin/${comp}" $core | sudo tee "$core.trace" 2>&1
            sudo ${GDB_CMD} "/opt/HPCCSystems/bin/${comp}" $core | sudo tee "$core.trace" 2>&1
    
       done

    else
        WriteLog "No core file generated." "${ML_TEST_LOG}"
    fi

    # Archive  logs
    pushd ${OBT_BIN_DIR}
    
    # Get tests stat
    if [[ ( $GET_STAT -eq 1 ) && ( -f QueryStat2.py) ]]
    then
        WriteLog "Get tests stat..." "${ML_TEST_LOG}"
        CMD="./QueryStat2.py -p ${HOME}/Perfstat/ -d '' -a --timestamp --compileTimeDetails 1 --graphTimings --allGraphItems --addHeader"
        WriteLog "  CMD: '$CMD'" "${ML_TEST_LOG}"
        ${CMD} >> ${ML_TEST_LOG} 2>&1
        retCode=$( echo $? )
        WriteLog "  RetCode: $retCode" "${ML_TEST_LOG}"
        WriteLog "  Files: $( ls -l perfstat* )" "${ML_TEST_LOG}"
        WriteLog "Done." "${ML_TEST_LOG}"
    else
        WriteLog "$OBT_BIN_DIR/QueryStat2.py not found. Skip perfromance result collection " "${ML_TEST_LOG}"
    fi

   
    if [[ $MAKE_ARCHIVE -eq 1 ]]
    then
        # Copy test summary to Wiki
        WriteLog "Copy ML test result files to ${TARGET_DIR}..." "${ML_TEST_LOG}"

        WriteLog "--->'${LOG_DIR}/ml-*.log'" "${ML_TEST_LOG}"
        WriteLog "--->$(ls -l ${LOG_DIR}/ml-* )" "${ML_TEST_LOG}"
        [ !  -d ${TARGET_DIR}/test/ ] && mkdir -p ${TARGET_DIR}/test
        res=$( cp -v ${LOG_DIR}/ml-*.log ${TARGET_DIR}/test/  2>&1 )
        WriteLog "---->res:${res}" "${ML_TEST_LOG}"

        WriteLog "--_>mltests.summary" "${ML_TEST_LOG}"
        res=$( cp -v mltests.summary ${TARGET_DIR}/test/mltests.summary 2>&1 )
        WriteLog "---->res:${res}" "${ML_TEST_LOG}"    
    
        WriteLog "Archive ${TARGET_PLATFORM} ML logs" "${ML_TEST_LOG}"
        ./archiveLogs.sh ml-${TARGET_PLATFORM} timestamp=${OBT_TIMESTAMP}
    fi
    
    popd
    #
    #---------------------------
    #
    # Uninstall HPCC to free as much disk space as can
    #

    if [[ ${KEEP_HPCC} -eq 0 ]]
    then
        WriteLog "Uninstall HPCC to free as much disk space as can on ${TARGET_PLATFORM}!" "${ML_TEST_LOG}"
    
        WriteLog "Uninstall HPCC-Platform" "${ML_TEST_LOG}"
    
        UninstallHPCC "${ML_TEST_LOG}" "${ML_WIPE_OFF_HPCC}"

    else
        WriteLog "Skip Uninstall HPCC on ${TARGET_PLATFORM} but stop it!" "${ML_TEST_LOG}"
        StopHpcc "${ML_TEST_LOG}"
    fi

    if [[ -f /etc/HPCCSystems/environment.xml.bak ]]
    then
        WriteLog "Restore original environment.xml on ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        
        ${SUDO} cp /etc/HPCCSystems/environment.xml.bak /etc/HPCCSystems/environment.xml
     
        if [ $? -ne 0 ]
        then
            WriteLog "Error in restore environment.xml file! ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        else
            WriteLog "The environment.xml file restored. ${TARGET_PLATFORM}" "${ML_TEST_LOG}"
        fi
    fi

    #
    #----------------------------
    #
    # Remove ECL-Bundles
    #
    #WriteLog "Remove ECL-Bundles" "${ML_TEST_LOG}"
    
    #rm -rf ${PERF_TEST_ROOT}/PerformanceTesting
    
    WriteLog "                                    " "${ML_TEST_LOG}"
    WriteLog "************************************" "${ML_TEST_LOG}"
    WriteLog " End of ML ${TARGET_PLATFORM} test. " "${ML_TEST_LOG}"
    WriteLog "                                    " "${ML_TEST_LOG}"
    
else
    WriteLog "                                   " "${ML_TEST_LOG}"
    WriteLog "***********************************" "${ML_TEST_LOG}"
    WriteLog " Skip ML ${TARGET_PLATFORM} test.  " "${ML_TEST_LOG}"
    WriteLog "                                   " "${ML_TEST_LOG}"

fi

#
#-----------------------------------------------------------------------------
#
# End of ML test
#


cd ${OBT_BIN_DIR}

#
#---------------------------
#
# Stop HPCC Systems
#

#WriteLog "Stop HPCC Systems ${TARGET_PLATFORM}" "${ML_TEST_LOG}"

#StopHpcc "${ML_TEST_LOG}"

WriteLog "End of Machine Learning test" "${ML_TEST_LOG}"

set +x