Run ISA tests in CI (#134)

* test: Ignore generated XML output

* run_tests: Build RVFI emulators too

Can't run tests with them though as they're built for direct instruction
injection (RVFI-DII) via an instruction stream over a network socket,
not fetching instructions from memory, so this remains just a build
test.

* run_tests/run_fp_tests: Print summary and give meaningful exit code

* run_tests/run_fp_tests: Include tests and failures in top-level XML entity

* run_tests/run_fp_tests: Use failure not error for XML output

The former is the standard tag for normal test failures, the latter is
for catastrophic things like test harness errors.

* Run ISA tests in CI

.github/workflows/compile.yml

@@ -6,10 +6,10 @@ jobs:
  build:
  runs-on: [ubuntu-18.04]
  steps:
- - name: Install opam2
- run: |
-  sudo add-apt-repository -y ppa:avsm/ppa
-  sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3
+ - name: Add opam2 PPA
+ run: sudo add-apt-repository -y ppa:avsm/ppa
+ - name: Install packages
+ run: sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 device-tree-compiler
  - name: Init opam
  run: opam init --disable-sandboxing -y
  - name: Install sail
@@ -18,7 +18,17 @@ jobs:
  uses: actions/checkout@HEAD
  with:
  submodules: true
- - name: Build RV32 simulators
- run: eval $(opam env) && make ARCH=RV32 -j2 csim rvfi osim
- - name: Build RV64 simulators
- run: eval $(opam env) && make ARCH=RV64 -j2 csim rvfi osim
+ - name: Build and test simulators
+ run: eval $(opam env) && test/run_tests.sh
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@v2
+ with:
+ name: tests.xml
+ path: test/tests.xml
+ - name: Upload event payload
+ if: always()
+ uses: actions/upload-artifact@v2
+ with:
+ name: event.json
+ path: ${{ github.event_path }}

.github/workflows/test-results.yml

@@ -0,0 +1,51 @@
+name: Publish test results
+
+on:
+ workflow_run:
+ workflows: ["CI"]
+ types:
+ - completed
+
+jobs:
+ publish-test-results:
+ runs-on: ubuntu-latest
+ if: github.event.workflow_run.conclusion != 'skipped'
+ steps:
+ - name: Download artifacts
+ uses: actions/github-script@v3.1.0
+ with:
+ script: |
+ var fs = require('fs');
+ var artifacts = await github.actions.listWorkflowRunArtifacts({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: ${{github.event.workflow_run.id }},
+ });
+ var matchArtifacts = artifacts.data.artifacts.filter((artifact) => {
+ return artifact.name == 'tests.xml' || artifact.name == 'event.json'
+ });
+ var count = matchArtifacts.length;
+ for (var i = 0; i < count; i++) {
+ var matchArtifact = matchArtifacts[i];
+ var download = await github.actions.downloadArtifact({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ artifact_id: matchArtifact.id,
+ archive_format: 'zip',
+ });
+ var name = matchArtifact.name;
+ var dest = name + '.zip'
+ fs.writeFileSync('${{github.workspace}}/' + dest, Buffer.from(download.data));
+ console.log("Downloaded", name, "as", dest);
+ }
+ - name: Extract test results
+ run: unzip tests.xml.zip
+ - name: Extract event payload
+ run: unzip event.json.zip
+ - name: Publish test results
+ uses: EnricoMi/publish-unit-test-result-action@v1
+ with:
+ commit: ${{ github.event.workflow_run.head_sha }}
+ event_file: event.json
+ event_name: ${{ github.event.workflow_run.event }}
+ files: tests.xml

test/.gitignore

@@ -0,0 +1 @@
+/tests.xml

test/run_fp_tests.sh

@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
  (( pass += 1 ))
  printf "$1: ${GREEN}$2${NC}\n"
- XML+=" <testcase name=\"$1\"/>\n"
+ SUITE_XML+=" <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
  (( fail += 1 ))
  printf "$1: ${YELLOW}$2${NC}\n"
- XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n"
+ SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n"
 }
 
 function red {
  (( fail += 1 ))
  printf "$1: ${RED}$2${NC}\n"
- XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n"
+ SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n"
 }
 
 function finish_suite {
  printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
- XML=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML </testsuite>\n"
- printf "$XML" >> $DIR/tests.xml
- XML=""
+ SUITES_XML+=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML </testsuite>\n"
+ SUITE_XML=""
+ (( all_pass += pass )) || :
+ (( all_fail += fail )) || :
  pass=0
  fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -85,4 +87,11 @@ for test in $DIR/riscv-tests/rv32u{f,d}*.elf $DIR/riscv-tests/rv32mi-p-csr.elf;
 done
 finish_suite "32-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+ exit 1
+fi

test/run_tests.sh

@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
  (( pass += 1 ))
  printf "$1: ${GREEN}$2${NC}\n"
- XML+=" <testcase name=\"$1\"/>\n"
+ SUITE_XML+=" <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
  (( fail += 1 ))
  printf "$1: ${YELLOW}$2${NC}\n"
- XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n"
+ SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n"
 }
 
 function red {
  (( fail += 1 ))
  printf "$1: ${RED}$2${NC}\n"
- XML+=" <testcase name=\"$1\">\n <error message=\"$2\">$2</error>\n </testcase>\n"
+ SUITE_XML+=" <testcase name=\"$1\">\n <failure message=\"$2\">$2</failure>\n </testcase>\n"
 }
 
 function finish_suite {
  printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
- XML=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML </testsuite>\n"
- printf "$XML" >> $DIR/tests.xml
- XML=""
+ SUITES_XML+=" <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML </testsuite>\n"
+ SUITE_XML=""
+ (( all_pass += pass )) || :
+ (( all_fail += fail )) || :
  pass=0
  fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -141,4 +143,33 @@ for test in $DIR/riscv-tests/rv64*.elf; do
 done
 finish_suite "64-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV32 make c_emulator/riscv_rvfi_RV32;
+then
+ green "Building 32-bit RISCV RVFI C emulator" "ok"
+else
+ red "Building 32-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "32-bit RISCV RVFI C tests"
+
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV64 make c_emulator/riscv_rvfi_RV64;
+then
+ green "Building 64-bit RISCV RVFI C emulator" "ok"
+else
+ red "Building 64-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "64-bit RISCV RVFI C tests"
+
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+ exit 1
+fi