Skip to content
This repository has been archived by the owner on Aug 5, 2022. It is now read-only.

Commit

Permalink
Merge pull request #102 from okartau/try-parallel-tests
Browse files Browse the repository at this point in the history 
CI Jenkinsfile,tester-exec: streamline, fix issue of lost test results
  • Loading branch information
@mythi
mythi authored Apr 10, 2017
2 parents adb4c8e + 15346e8 commit aa51d87
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 71 deletions.
120 changes: 64 additions & 56 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,63 +122,47 @@ try {
def test_runs = [:]
for(int i = 0; i < test_devices.size(); i++) {
def test_device = test_devices[i]
// only if built for machine that this tester wants
if ( target_machine == mapping["${test_device}"] ) {
// testinfo_data may contain multiple lines stating different images
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
for (int m = 0; m < separated_testinfo.length; m++) {
def one_image_testinfo = separated_testinfo[m]
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
test_runs["test_${m}_${test_device}"] = {
node('refkit-tester') {
deleteDir() // clean workspace
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
writeFile file: 'tester-exec.sh', text: tester_script
// append newline so that tester-exec.sh can parse it using "read"
one_image_testinfo += "\n"
// write testinfo file on this tester for this image, one line per tester
writeFile file: "testinfo.csv", text: one_image_testinfo
String[] one_testinfo_elems = one_image_testinfo.split(",")
def img = one_testinfo_elems[0]
try {
withEnv(["CI_BUILD_ID=${ci_build_id}",
"MACHINE=${mapping["${test_device}"]}",
"TEST_DEVICE=${test_device}" ]) {
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
}
} catch (Exception e) {
throw e
} finally {
// read tests summary prepared by tester-exec.sh
// Here one tester adds it's summary piece to the global buffer.
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
archiveArtifacts allowEmptyArchive: true,
artifacts: '**/*.log, **/*.xml, **/aft-results*.tar.bz2'
// only if built for machine that this tester wants
if ( target_machine == mapping["${test_device}"] ) {
// testinfo_data may contain multiple lines stating different images
String[] separated_testinfo = testinfo_data["${target_machine}"].split("\n")
for (int m = 0; m < separated_testinfo.length; m++) {
def one_image_testinfo = separated_testinfo[m]
echo "Image #${m} to be tested on test_${test_device} info: ${separated_testinfo[m]}"
test_runs["test_${m}_${test_device}"] = {
node('refkit-tester') {
deleteDir() // clean workspace
echo "Testing test_${test_device} with image_info: ${one_image_testinfo}"
writeFile file: 'tester-exec.sh', text: tester_script
// append newline so that tester-exec.sh can parse it using "read"
one_image_testinfo += "\n"
// create testinfo.csv on this tester describing one image
writeFile file: "testinfo.csv", text: one_image_testinfo
def img = one_image_testinfo.split(",")[0]
try {
withEnv(["CI_BUILD_ID=${ci_build_id}",
"MACHINE=${mapping["${test_device}"]}",
"TEST_DEVICE=${test_device}" ]) {
sh 'chmod a+x tester-exec.sh && ./tester-exec.sh'
}
step([$class: 'XUnitPublisher',
testTimeMargin: '3000',
thresholdMode: 1,
thresholds: [
[$class: 'FailedThreshold',
failureNewThreshold: '0',
failureThreshold: '0',
unstableNewThreshold: '99999',
unstableThreshold: '99999'],
[$class: 'SkippedThreshold',
failureNewThreshold: '99999',
failureThreshold: '99999',
unstableNewThreshold: '99999',
unstableThreshold: '99999']],
tools: [[$class: 'JUnitType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: 'TEST-*.xml',
skipNoTestFiles: false,
stopProcessingIfError: true]]])
} // node
} // test_runs =
} // for m
} // if target_machine == mapping
} catch (Exception e) {
throw e
} finally {
// read tests summary prepared by tester-exec.sh
// Here one tester adds it's summary piece to the global buffer.
global_sum_log += readFile "results-summary-${test_device}.${img}.log"
archiveArtifacts allowEmptyArchive: true,
artifacts: '*.log, *.xml'
}
// without locking we may lose tester result set(s)
// if testers run xunit step in nearly same time
lock(resource: "step-xunit") {
step_xunit()
}
} // node
} // test_runs =
} // for
} // if target_machine == mapping
} // for i
stage('Parallel test run') {
set_gh_status_pending(is_pr, 'Testing')
Expand Down Expand Up @@ -300,3 +284,27 @@ def set_gh_status_pending(is_pr, _msg) {
setGitHubPullRequestStatus state: 'PENDING', context: "${env.JOB_NAME}", message: "${_msg}"
}
}

def step_xunit() {
step([$class: 'XUnitPublisher',
testTimeMargin: '3000',
thresholdMode: 1,
thresholds: [
[$class: 'FailedThreshold',
failureNewThreshold: '0',
failureThreshold: '0',
unstableNewThreshold: '99999',
unstableThreshold: '99999'],
[$class: 'SkippedThreshold',
failureNewThreshold: '99999',
failureThreshold: '99999',
unstableNewThreshold: '99999',
unstableThreshold: '99999']],
tools: [
[$class: 'JUnitType',
deleteOutputFiles: true,
failIfNotNew: true,
pattern: 'TEST-*.xml',
skipNoTestFiles: false,
stopProcessingIfError: true]]])
}
24 changes: 9 additions & 15 deletions docker/tester-exec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ testimg() {
_IMG_NAME=$1
TEST_SUITE_FILE=$2
TEST_CASES_FILE=$3
_IMG_NAME_MACHINE=${_IMG_NAME}-${MACHINE}

# Get test suite
wget ${_WGET_OPTS} ${TEST_SUITE_FOLDER_URL}/${_IMG_NAME}/${TEST_SUITE_FILE}
Expand All @@ -34,21 +33,16 @@ testimg() {
cp $HOME/.config.ini.wlan ${_WLANCONF}
chmod 644 ${_WLANCONF}

FN_BASE=${_IMG_NAME_MACHINE}-${CI_BUILD_ID}
FILENAME=${FN_BASE}.wic
FILENAME_BMAP=${FILENAME}.bmap
FILENAME_XZ=${FILENAME}.xz
FILENAME_ZIP=${FILENAME}.zip

FILENAME=${_IMG_NAME}-${MACHINE}-${CI_BUILD_ID}.wic
set +e
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_BMAP}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_XZ} -O - | unxz - > ${FILENAME}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME}
if [ ! -s ${FILENAME} ]; then
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME_ZIP}
if [ -s ${FILENAME_ZIP} ]; then
unzip ${FILENAME_ZIP}
wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip
if [ -s ${FILENAME}.zip ]; then
unzip ${FILENAME}.zip
else
echo "ERROR: No file ${FILENAME_XZ} or ${FILENAME_ZIP} found, can not continue."
echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue."
exit 1
fi
fi
Expand Down Expand Up @@ -76,6 +70,8 @@ testimg() {
daft ${DEVICE} ${FILENAME} --record
AFT_EXIT_CODE=$?

# delete symlinks, these point outside of local set and are useless
find . -type l -print -delete
# modify names inside TEST-*.xml files to contain device and img_name
# as these get shown in same xUnit results table in Jenkins
sed -e "s/name=\"oeqa/name=\"${DEVICE}.${_IMG_NAME}.oeqa/g" -i TEST-*.xml
Expand Down Expand Up @@ -107,8 +103,6 @@ testimg() {
echo " Run rate:${run_rate}% Pass rate of total:${pass_rate_of_total}% Pass rate of exec:${pass_rate_of_exec}%" >> $sumfile
fi
echo "-------------------------------------------------------------------" >> $sumfile
# combine artifacts into single file for easier download
tar c --ignore-failed-read results* *.xml *.log | bzip2 -c9 > aft-results_${DEVICE}_${_IMG_NAME}_${TEST_SUITE_FILE}.tar.bz2
set -e

return ${AFT_EXIT_CODE}
Expand Down

0 comments on commit aa51d87

Please sign in to comment.