Github action for CMSIS-DSP testing (#183)

Add github action for running CMSIS-DSP tests on each commit or PR to the main branch

.github/workflows/runtest.yaml

@@ -0,0 +1,86 @@
+name: Compile and Run
+on:
+ workflow_dispatch:
+ pull_request:
+ branches: [main]
+ push:
+ branches: [main]
+
+permissions: 
+ actions: read
+ security-events: write
+
+jobs:
+ CI_test_run: 
+ runs-on: ubuntu-latest 
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Python 3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10' 
+
+ - name: Install system packages
+ run: |
+ sudo add-apt-repository ppa:deadsnakes/ppa
+ sudo apt-get install libpython3.9 libtinfo5
+
+ - name: Activate vcpkg
+ uses: ARM-software/cmsis-actions/vcpkg@v1
+
+ - name: Activate Arm tool license
+ uses: ARM-software/cmsis-actions/armlm@v1
+
+ - name: Prepare framework
+ run: |
+ cd Testing
+ echo "Create missing folders"
+ mkdir FullBenchmark
+ mkdir Output
+ mkdir GeneratedInclude
+ mkdir GeneratedSource
+ mkdir GeneratedIncludeBench
+ mkdir GeneratedSourceBench
+
+ echo "Install missing python packages"
+ pip install -r requirements.txt
+
+ echo "Preprocess test description"
+ python preprocess.py -f desc.txt -o Output.pickle
+ python preprocess.py -f desc_f16.txt -o Output_f16.pickle
+
+ echo "Generate missing CPP headers"
+ python processTests.py -gen . -p Patterns -d Parameters -f Output.pickle -e
+ python processTests.py -gen . -p Patterns -d Parameters -f Output_f16.pickle -e
+
+ cd cmsis_build
+
+ echo "Load missing pack"
+ csolution list packs -s test_ac6.csolution.yml -m > required_packs.txt
+ 
+ cat required_packs.txt
+ cpackget add -f required_packs.txt
+
+ - name: Execute
+ run: |
+ cd Testing/cmsis_build
+ echo "Running tests"
+
+ python runall.py -avh $AVH_FVP_PLUGINS/../bin
+
+ - name: Upload test report
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-report
+ path: Testing/cmsis_build/summary.html
+
+
+ - name: Check error
+ run: |
+ cd Testing/cmsis_build
+ 
+ echo "Checking output..."
+ test "$(grep "Error" summary.html | wc -l)" -eq 0

Testing/cmsis_build/runall.py

@@ -12,9 +12,11 @@
 parser.add_argument('-n', action='store_true', help="No force rebuild")
 parser.add_argument('-r', action='store_true', help="Raw results only")
 parser.add_argument('-c', action='store_true', help="Display cycles (so passing test are displayed)")
+parser.add_argument('-l', action='store_true', help="Local run (not github action)")
 
 args = parser.parse_args()
 
+
 DEBUG=False 
 if args.d:
  DEBUG=True
@@ -112,9 +114,19 @@ def run(*args,mustPrint=False,dumpStdErr=True):
 }
 
 # Windows executable
-# (At some point this script will also support
-# unix)
-avhExe={
+avhUnixExe={
+ "CS310":"FVP_Corstone_SSE-310_Ethos-U65",
+ "CS300":"FVP_Corstone_SSE-300_Ethos-U55",
+ "M55":"FVP_MPS2_Cortex-M55",
+ "M33_DSP_FP":"FVP_MPS2_Cortex-M33",
+ "M7DP":"FVP_MPS2_Cortex-M7",
+ "M4FP":"FVP_MPS2_Cortex-M4",
+ "M3":"FVP_MPS2_Cortex-M3",
+ "M23":"FVP_MPS2_Cortex-M23",
+ "M0plus":"FVP_MPS2_Cortex-M0plus",
+}
+
+avhWindowsExe={
  "CS310":"VHT_Corstone_SSE-310.exe",
  "CS300":"VHT_Corstone_SSE-300_Ethos-U55.exe",
  "M55":"VHT_MPS2_Cortex-M55.exe",
@@ -138,7 +150,20 @@ def runAVH(build,core):
  if os.path.exists(elf):
  app = elf
  config = os.path.join("configs",configFiles[core])
- avh = os.path.join(AVHROOT,avhExe[core])
+
+ if AVHROOT:
+ avhAttempt = os.path.join(AVHROOT,avhWindowsExe[core])
+ if os.path.exists(avhAttempt):
+ avh = avhAttempt
+
+ avhAttempt = os.path.join(AVHROOT,avhUnixExe[core])
+ if os.path.exists(avhAttempt):
+ avh = avhAttempt
+ else:
+ avh = avhUnixExe[core]
+
+
+
  res=run(avh,"-f",config,app)
  return(res)
 
@@ -160,50 +185,14 @@ def runAVH(build,core):
 # Test suite and output pickle needed to decode the result
 #print(allSuites)
 
-#allSuites=[
-#("MFCCQ15","../Output.pickle"),
-#("MFCCQ31","../Output.pickle"),
-#("SupportTestsF16","../Output_f16.pickle"),
-#]
 
-#allSuites=[("ComplexTestsF32","../Output.pickle"),
-#("DistanceTestsF32","../Output.pickle"),
-#("UnaryTestsF32","../Output.pickle"),
-#("QuaternionTestsF32","../Output.pickle"),
-#("StatsTestsF32","../Output.pickle")
-#]
 
 #allSuites=[
-#("TransformCF64","../Output.pickle"),
-#("TransformCF32","../Output.pickle"),
-#("TransformCQ31","../Output.pickle"),
-#("TransformCQ15","../Output.pickle"),
-#
-#("TransformRF64","../Output.pickle"),
-#("TransformRF32","../Output.pickle"),
-#("TransformRQ31","../Output.pickle"),
-#("TransformRQ15","../Output.pickle"),
-#
-#("MFCCF32","../Output.pickle"),
-#("MFCCQ31","../Output.pickle"),
-#("MFCCQ15","../Output.pickle"),
-#
-#("MFCCF16","../Output_f16.pickle"),
-#("TransformCF16","../Output_f16.pickle"),
-#("TransformRF16","../Output_f16.pickle")
+#("DECIMF64","../Output.pickle"),
+#("UnaryTestsF32","../Output.pickle"),
+#("UnaryTestsF16","../Output_f16.pickle"),
 #]
 
-#allSuites=[("StatsTestsQ7","../Output.pickle")]
-
-allSuites=[
-("UnaryTestsF64","../Output.pickle"),
-("UnaryTestsF32","../Output.pickle"),
-("UnaryTestsF16","../Output_f16.pickle"),
-#("MISCQ15","../Output.pickle"),
-#("MISCQ7","../Output.pickle"),
-#("FIRF16","../Output_f16.pickle")
-]
-
 # Solution and build file for all
 # the tests
 # It is a pair : csolution target type and AVH identification
@@ -213,7 +202,7 @@ def runAVH(build,core):
  'test_ac6.csolution.yml':[
  # ("VHT-Corstone-310","CS310"),
  ("VHT-Corstone-300","CS300"),
- # ("VHT-Corstone-300-NOMVE","CS300"),
+ ("VHT-Corstone-300-NOMVE","CS300"),
  ("VHT_M33","M33_DSP_FP"),
  ("VHT_M7","M7DP"),
  ("VHT_M7_UNROLLED","M7DP"),
@@ -222,24 +211,31 @@ def runAVH(build,core):
  # #("VHT_M23","M23"),
  ("VHT_M0P","M0plus")
  ],
- 'test_gcc.csolution.yml':[
-  #("VHT-Corstone-310","CS310"),
+ #'test_gcc.csolution.yml':[
+ # ("VHT-Corstone-300","CS300"),
  #("VHT_M55","M55"),
  ##("VHT_M33","M33_DSP_FP"),
- ("VHT_M7","M7DP"),
- ("VHT_M7_UNROLLED","M7DP"),
- ("VHT_M4","M4FP"),
+ # ("VHT_M7","M7DP"),
+ # ("VHT_M7_UNROLLED","M7DP"),
+ # ("VHT_M4","M4FP"),
  ##("VHT_M3","M3"),
  ##("VHT_M23","M23"),
- ("VHT_M0P","M0plus")
- ]
+ # ("VHT_M0P","M0plus")
+ #]
 }
 
 # Override previous solutions for more restricted testing.
 #solutions={
 # 'test_ac6.csolution.yml':[
-# #("VHT-Corstone-300","CS300"),
-# ("VHT_M7_UNROLLED","M7DP"),
+# ("VHT-Corstone-300","CS300"),
+# #("VHT_M7_UNROLLED","M7DP"),
+# ]
+#}
+#
+#solutions={
+# 'test_gcc.csolution.yml':[
+# ("VHT-Corstone-300","CS300"),
+# #("VHT_M7_UNROLLED","M7DP"),
 # ]
 #}
 
@@ -336,7 +332,8 @@ def runAVH(build,core):
  print(res.msg,file=f)
  print(HTMLFOOTER,file=f)
 
-if ERROR_OCCURED:
- sys.exit("Error occurred")
-else:
- sys.exit(0)
+if args.l:
+ if ERROR_OCCURED:
+ sys.exit("Error occurred")
+ else:
+ sys.exit(0)