Implement CI for 64-bit CVA6 (#452)

This PR extends our CI system to also run tests on real CVA6 hardware,
to ensure that we don't inadvertently break this platform. The CI
infrastructure depends on a bitstream version that includes our changes
to the bootrom (openhwgroup/cva6#2267), which
implement a simple UART-based firmware flashing functionality.

As part of extending our CI to this platform, we spent some effort to
make the system hostable and extensible by others. We check in some
previously unofficial helper scripts that operate an FT245R relay board,
enabling us to turn boards off and on remotely. We also include some
infrastructure for discovering serial ports (since our runner machine
has many of these).

.github/workflows/main.yml

@@ -222,7 +222,7 @@ jobs:
  # Combine cache directories to save space
  combine-caches:
  runs-on: ubuntu-latest
- if: ${{ always() }}
+ if: success() || failure()
  needs: build
  steps:
  - name: Install dependencies
@@ -378,6 +378,6 @@ jobs:
  uses: ./.github/workflows/build-runtime.yml
 
  # System tests, which are run for simulatable and self-hostable platforms
- test-system:
+ test-system-functionality:
  needs: build
  uses: ./.github/workflows/test-system.yml

.github/workflows/test-system.yml

@@ -3,19 +3,31 @@ on:
  workflow_call:
 
 jobs:
- test-generic:
- runs-on: ubuntu-latest
+ test-system:
+ runs-on: ${{ matrix.platform == 'generic' && 'ubuntu-latest' || matrix.platform }}
+ environment: ${{ matrix.platform != 'generic' && 'track' || null }}
  strategy:
+ fail-fast: false
  matrix:
- platform: [generic]
+ platform: [generic, mpfs, cva6]
  bits: [32, 64]
+ exclude:
+ # mpfs is not 32 bit
+ - platform: mpfs
+ bits: 32
+ # ignore 32-bit cva6 for now
+ - platform: cva6
+ bits: 32
 
  steps:
- # We don't need submodules here since Keystone is a monorepo!
  - name: Checkout Keystone
  uses: actions/checkout@v4
  with:
- submodules: 'false'
+ submodules: 'true'
+ sparse-checkout: |
+ .
+ scripts/
+ mkutils/
 
  - name: Restore build directory
  uses: actions/download-artifact@v4
@@ -26,188 +38,63 @@ jobs:
  - name: Decompress build directory
  run: cat build.tar.xz | xz -d -T0 | tar -xf -
 
- - name: Test Keystone system
- run: |
- # Fix permissions on the key
- chmod 600 build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/target/root/.ssh/id-rsa
-
- # Launch QEMU
- export KEYSTONE_PLATFORM=${{ matrix.platform }}
- export KEYSTONE_BITS=${{ matrix.bits }}
- export QEMU_PORT=$(( RANDOM + 1024 ))
- export LD_LIBRARY_PATH=build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/host/lib
- screen -L -dmS qemu bash -c "make run 2>&1 | tee run.log"
-
- # TODO: check for connectivity instead of sleeping
- sleep 60
-
- export CALL_LOGFILE=cmd.log
- echo "" > $CALL_LOGFILE
-
- KEYSTONE_COMMAND="modprobe keystone-driver" make call
- KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
- KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
- KEYSTONE_COMMAND="poweroff" make call
-
- - name: Check expected
- run: |
- [[ -z $(diff cmd.log scripts/ci/expected.log) ]]
-
- - name: Upload run log
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
- path: run.log
-
- - name: Upload cmd log
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
- path: cmd.log
-
- test-mpfs:
- runs-on: [self-hosted, mpfs]
- environment: track
- steps:
- # We don't need submodules here since Keystone is a monorepo!
- - name: Checkout Keystone
- uses: actions/checkout@v4
- with:
- submodules: 'false'
-
- - name: Restore build directory
- uses: actions/download-artifact@v4
- with:
- name: keystone-mpfs64-builddir
- path: .
-
- - name: Decompress build directory
- run: cat build.tar.xz | xz -d -T0 | tar -xf -
-
- # Test the firmware, first by flashing it
- - name: Flash HSS
+ - name: Flash and check firmware
  env:
- POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
- POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
- SC_INSTALL_DIR: ${{ vars.SC_INSTALL_DIR }}
+ KEYSTONE_PLATFORM: ${{ matrix.platform }}
+ KEYSTONE_BITS: ${{ matrix.bits }}
+ LOGFILE: fw-program.log
  run: |
- $POWER_ON_CMD
- export FPGENPROG=$(which fpgenprog)
- make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null >program.log
- $POWER_OFF_CMD
-
- # Check if we succeeded
- [[ ! -z $(cat program.log | grep "mpfsBootmodeProgrammer completed successfully") ]]
+ if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh ]]; then
+ scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh
+ fi
 
- - name: Upload HSS program log
- if: failure()
+ - name: Upload firmware programming log
+ if: ${{ matrix.platform != 'generic' && failure() }}
  uses: actions/upload-artifact@v4
  with:
- name: test-keystone-mpfs64-prog-hss.log
- path: program.log
+ name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-fw-program.log
+ path: fw-program.log
 
- # And then verifying that we can actually get to the command line
- - name: Check HSS ok
+ - name: Flash and check OS
  env:
- POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
- POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
- FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+ KEYSTONE_PLATFORM: ${{ matrix.platform }}
+ KEYSTONE_BITS: ${{ matrix.bits }}
+ LOGFILE: os-program.log
  run: |
- # Collect serial output
- TTYDEV=$($FIND_TTY_CMD 0)
- screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run-hss.log"
- $POWER_ON_CMD ; sleep 30 ; $POWER_OFF_CMD
- screen -XS mpfs-tty quit
-
- # At least the first hart should have started
- [[ ! -z $(cat run-hss.log | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change: \[Running\]") ]]
+ if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-os.sh ]]; then
+ scripts/ci/plat/${{ matrix.platform }}/flash-os.sh
+ fi
 
- - name: Upload HSS run log
- if: failure()
+ - name: Upload OS programming log
+ if: ${{ matrix.platform != 'generic' && failure() }}
  uses: actions/upload-artifact@v4
  with:
- name: test-keystone-mpfs64-run-hss.log
- path: run-hss.log
-
- # Now we also need to flash the disk. First, get into usbdmsc
- - name: Flash OS
- env:
- POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
- POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
- FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
- run: |
- # Wait for the board to come up a bit. We'll hammer it with serial
- # input to ensure that we halt the boot at HSS
- TTYDEV=$($FIND_TTY_CMD 0)
- $POWER_ON_CMD
- NOW=$(date +%s)
- stty raw -echo 115200 < "$TTYDEV"
- while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done
-
- echo "" > "$TTYDEV"
- echo "usbdmsc" > "$TTYDEV"
-
- # Wait a bit for the USB to connect then flash
- sleep 10
- FOUND_DEVICE=""
- for d in /dev/sd? ; do
- if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then
- FOUND_DEVICE="yes"
- dd if=build-mpfs64/buildroot.build/images/sdcard.img of="$d" bs=4M oflag=direct
- break
- fi
- done
-
- $POWER_OFF_CMD
- [[ ! -z "$FOUND_DEVICE" ]]
+ name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-os-program.log
+ path: os-program.log
 
  - name: Test Keystone system
  env:
- POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
- POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
- FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
- KEYSTONE_IP: ${{ vars.BOARD_IP_MPFS }}
+ KEYSTONE_PLATFORM: ${{ matrix.platform }}
+ KEYSTONE_BITS: ${{ matrix.bits }}
+ LOGFILE: run.log
+ CMD_LOGFILE: cmd.log
  run: |
- # Fix permissions on the key
- chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa
-
- # Start the board
- TTYDEV=$($FIND_TTY_CMD 1)
- export KEYSTONE_PLATFORM=mpfs
- export KEYSTONE_BITS=64
- screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run.log"
- $POWER_ON_CMD
-
- # TODO: check for connectivity instead of sleeping
- sleep 60
-
- export CALL_LOGFILE=cmd.log
- echo "" > $CALL_LOGFILE
-
- KEYSTONE_COMMAND="modprobe keystone-driver" make call
- KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
- # Todo: attestation does not yet work in mpfs
- #KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
-
- $POWER_OFF_CMD
- screen -XS mpfs-tty quit
+ scripts/ci/plat/${{ matrix.platform }}/test.sh
 
  - name: Check expected
  run: |
- [[ -z $(diff cmd.log scripts/ci/expected-mpfs.log) ]]
+ [[ -z $(diff -wB cmd.log scripts/ci/plat/${{ matrix.platform }}/expected.log) ]]
 
  - name: Upload run log
  if: failure()
  uses: actions/upload-artifact@v4
  with:
- name: test-keystone-mpfs64-run.log
+ name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
  path: run.log
 
  - name: Upload cmd log
  if: failure()
  uses: actions/upload-artifact@v4
  with:
- name: test-keystone-mpfs64-cmd.log
+ name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
  path: cmd.log

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "overlays/microchip"]
  path = overlays/microchip
  url = https://github.com/linux4microchip/buildroot-external-microchip
+[submodule "scripts/ci/utils/relay_ft245r"]
+ path = scripts/ci/utils/relay_ft245r
+ url = https://github.com/vpatron/relay_ft245r

mkutils/plat/cva6/run.mk

@@ -5,16 +5,30 @@
 
 
 PAYLOAD = $(BUILDROOT_BUILDDIR)/images/fw_payload.bin
-SDDEVICE_PART1 = $(SD_DEVICE)1
-SDDEVICE_PART2 = $(SD_DEVICE)2
-
-
+KERNEL = $(BUILDROOT_BUILDDIR)/images/uImage
+SDDEVICE_PART1 = $(shell lsblk $(SD_DEVICE) -no PATH | head -2 | tail -1)
+SDDEVICE_PART2 = $(shell lsblk $(SD_DEVICE) -no PATH | head -3 | tail -1)
 
 flash: $(SD_DEVICE)
  $(info PAYLOAD INFORMATION)
  $(info $(PAYLOAD))
  $(info $(SD_DEVICE))
  $(info $(SDDEVICE_PART1))
  $(info $(SDDEVICE_PART2))
- sgdisk --clear -g --new=1:2048:40M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
+ sgdisk --clear -g --new=1:2048:4M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
  dd if=$(PAYLOAD) of=$(SDDEVICE_PART1) status=progress oflag=sync bs=1M
+ dd if=$(KERNEL) of=$(SDDEVICE_PART2) status=progress oflag=sync bs=1M
+
+CALL_LOGFILE ?= $(shell mktemp)
+call:
+ $(call log,info,Calling command on the CVA6 board)
+ ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
+ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+ root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \
+ grep -v "Warning: Permanently added" | tee -a $(CALL_LOGFILE)
+
+debug-connect:
+ $(call log,info,Connecting to OpenOCD)
+ $(BUILDROOT_BUILDDIR)/host/bin/riscv64-buildroot-linux-gnu-gdb \
+ -iex "set KEYSTONE=$(KEYSTONE)" \
+ -x $(KEYSTONE)/scripts/gdb/cva6.cfg

mkutils/plat/mpfs/run.mk

@@ -20,7 +20,7 @@ run:
 
 CALL_LOGFILE ?= $(shell mktemp)
 call:
- $(call log,info,Calling command in QEMU)
+ $(call log,info,Calling command on the MPFS board)
  ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
  root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \