fpga: Configurable remote boot

target/fpga/Makefile

@@ -7,12 +7,11 @@
 MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MKFILE_DIR  := $(dir $(MKFILE_PATH))
 ROOT        := ${MKFILE_DIR}../../../..
-CVA6_SDK    ?= ${ROOT}/../cva6-sdk
+CVA6_SDK    ?=
 DEBUG       ?= 0
 EXT_JTAG    ?= 0
-VCU         ?= 01
-FPGA_ID     :=
-HW_SERVER   :=
+FPGA_PATH   := $(XILINX_FPGA_PATH)
+HW_SERVER   := $(XILINX_HOST):$(XILINX_PORT)
 BENDER      ?= bender
 VIVADO      ?= vitis-2020.2 vivado
 # Do not proceed with implem (CI)
@@ -54,12 +53,12 @@ program:
 	${VIVADO} -mode batch -source occamy_vcu128_program.tcl -tclargs ${VCU}
 
 flash: ${FILE}
-	${VIVADO} -mode batch -source occamy_vcu128_flash.tcl -tclargs ${HW_SERVER} ${FPGA_ID} flash.mcs ${OFFSET} ${FILE}
+	${VIVADO} -mode batch -source occamy_vcu128_flash.tcl -tclargs ${HW_SERVER} ${FPGA_PATH} flash.mcs ${OFFSET} ${FILE}
 	rm flash.mcs
 
 # Flash only uboot (made for TFTP bootmode), overwrite the bootrom, and reset the board
 flashrun: ${UBOOT_ITB}
-	${VIVADO} -mode batch -source occamy_vcu128_flashrun.tcl -tclargs ${HW_SERVER} ${FPGA_ID} flash.mcs 6000000 ${UBOOT_ITB}
+	${VIVADO} -mode batch -source occamy_vcu128_flashrun.tcl -tclargs ${HW_SERVER} ${FPGA_PATH} flash.mcs 6000000 ${UBOOT_ITB}
 	rm flash.mcs
 
 flash-u-boot:

target/fpga/bootrom/Makefile

@@ -38,8 +38,8 @@ all: bootrom-spl.coe bootrom.tcl bootrom-spl.tcl
 	$(CC) $(CFLAGS) -DGIT_SHA=\"$(GIT_SHA)\"  $(INCLUDES) -c $<  -o $@
 	@echo "CC    <= $<"
 
-%.dtb: %.dts
-	dtc -I dts $< -O dtb -o $@
+%.dtb: %.dts $(wildcard *.dtsi)
+	dtc -@ -I dts $< -O dtb -o $@
 
 bootrom.elf bootrom.dump bootrom.bin: bootrom.S $(OBJS_C) bootrom.ld occamy.dtb
 	$(CC) $(CFLAGS) $(INCLUDES) -Tbootrom.ld $< $(OBJS_C) -o bootrom.elf

target/fpga/bootrom/mac_address.dtsi

@@ -0,0 +1,2 @@
+local-mac-address = [ 00 00 00 00 00 00 ];
+mac-address = [ 00 00 00 00 00 00 ];

target/fpga/bootrom/occamy.dts

@@ -191,8 +191,8 @@
       clock-names = "s_axi_lite_clk", "axis_clk";
       // interrupt and mac_irq
       interrupts-extended = <&PLIC0 1 &PLIC0 6>;
-      local-mac-address = [ 00 0A 35 07 D5 DD ]; // hero-vcu128-03
-      mac-address = [ 00 0A 35 07 D5 DD ]; // hero-vcu128-03
+      /include/ "mac_address.dtsi"
+      /include/ "remote_boot.dtsi"
       device_type = "network";
       axistream-connected = <&eth_dma0>;
       axistream-control-connected = <&eth_dma0>;

target/fpga/bootrom/occamy_pcie.dts

@@ -117,9 +117,15 @@
         reg-names = "quadrant-control";
         reg = <0x0 0x0b000000 0x0 0x10000>;
       };
-      scratchpad-narrow@70000000 {
-        compatible = "eth,scratchpad-narrow";
-        reg = <0x0 0x70000000 0x0 0x80000>;
+      // We do not use the spm-narrow (contains OpenSBI code)
+      spm_wide: spm-wide@71000000 {
+        compatible = "eth,occamy-spm-wide";
+        reg-names = "spm-wide";
+        reg = <0x0 0x71000000 0x0 0x100000>;
+      };
+      pcie_axi_bar_mem: pcie-axi-bar-mem@20000000 {
+        compatible = "eth,pcie-axi-bar-mem";
+        reg = <0x0 0x20000000 0x0 0x40000000>;
       };
       // Instantiate a snitch cluster
       snitch-cluster@10000000 {

target/fpga/bootrom/src/main.c

@@ -79,7 +79,7 @@ int main() {
             // Copy the DBT at (SPM+4)
 
             print_uart("\r\nCopying DTB at ");
-            print_uart_addr(SPL_DEST + 1);
+            print_uart_addr(SPL_DEST + 4);
             for (int i = 0; i < totalsize; i++)
                 *(uint8_t *)(SPL_DEST + 4 + i) =
                     *(((uint8_t *)__dtb_start) + i);

target/fpga/occamy_vcu128_bd.tcl

@@ -710,8 +710,15 @@ proc create_root_design { parentCell } {
    CONFIG.axist_bypass_scale {Gigabytes} \
    CONFIG.axist_bypass_size {4} \
    CONFIG.axisten_freq {125} \
+   CONFIG.bar_indicator {BAR_1:0} \
    CONFIG.functional_mode {AXI_Bridge} \
+   CONFIG.pf0_bar0_64bit {true} \
+   CONFIG.pf0_bar0_prefetchable {true} \
+   CONFIG.pf0_bar0_scale {Gigabytes} \
+   CONFIG.pf0_bar0_size {4} \
    CONFIG.pf0_device_id {9014} \
+   CONFIG.pf0_msix_cap_pba_bir {BAR_1:0} \
+   CONFIG.pf0_msix_cap_table_bir {BAR_1:0} \
    CONFIG.pl_link_cap_max_link_width {X4} \
    CONFIG.xdma_axi_intf_mm {AXI_Memory_Mapped} \
    CONFIG.xdma_axilite_slave {true} \
@@ -874,7 +881,7 @@ proc create_root_design { parentCell } {
   assign_bd_address -offset 0x4CC00000 -range 0x00400000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs hbm_0/SAPB_0/Reg] -force
   assign_bd_address -offset 0x4C800000 -range 0x00400000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs hbm_0/SAPB_1/Reg] -force
   assign_bd_address -offset 0x20000000 -range 0x20000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_B/BAR0] -force
-  assign_bd_address -offset 0x00000000 -range 0x20000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_LITE/CTL0] -force
+  assign_bd_address -offset 0x4E000000 -range 0x01000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_LITE/CTL0] -force
   assign_bd_address -offset 0x00000000 -range 0x0001000000000000 -target_address_space [get_bd_addr_spaces xdma_0/M_AXI_B] [get_bd_addr_segs occamy/s_axi_pcie/reg0] -force
 
   # Exclude Address Segments

target/fpga/occamy_vcu128_flash.tcl

@@ -8,17 +8,17 @@
 # Programs the SPI Flash of the VCU128 board with with two partitions
 # 
 # HW_SERVER  host:port URL to the server where the FPGA board is connected to 
-# FPGA_ID    Serial of the FPGA to target
+# FPGA_PATH  Serial of the FPGA to target
 # MCS        Output flash configuration file
 # OFFSET0    Address offset of partition 0
 # FILE0      File to program to partition 0
 
 # Parse arguments
 if {$argc < 5} {
-    error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_ID MCS OFFSET0 FILE0"
+    error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_PATH MCS OFFSET0 FILE0"
 }
 set HW_SERVER [lindex $argv 0]
-set FPGA_ID   [lindex $argv 1]
+set FPGA_PATH   [lindex $argv 1]
 set MCS       [lindex $argv 2]
 set OFFSET0   [lindex $argv 3]
 set FILE0     [lindex $argv 4]
@@ -38,8 +38,8 @@ write_cfgmem -force -format mcs -size 256 -interface SPIx4 \
 # Open and connect HW manager
 open_hw_manager
 connect_hw_server -url ${HW_SERVER} -allow_non_jtag
-current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}]
-set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}]
+current_hw_target [get_hw_targets ${FPGA_PATH}]
+set_property PARAM.FREQUENCY 15000000 [get_hw_targets ${FPGA_PATH}]
 open_hw_target
 current_hw_device [get_hw_devices xcvu37p_0]
 

target/fpga/occamy_vcu128_flashrun.tcl

@@ -9,7 +9,7 @@
 # Afterwards programs the real bitstream and runs the bootrom
 # 
 # HW_SERVER  host:port URL to the server where the FPGA board is connected to 
-# FPGA_ID    Serial of the FPGA to target
+# FPGA_PATH  Serial of the FPGA to target
 # MCS        Output flash configuration file
 # OFFSET0    Address offset of partition 0
 # FILE0      File to program to partition 0
@@ -18,10 +18,10 @@ source occamy_vcu128_procs.tcl
 
 # Parse arguments
 if {$argc < 5} {
-    error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_ID MCS OFFSET0 FILE0"
+    error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_PATH MCS OFFSET0 FILE0"
 }
 set HW_SERVER [lindex $argv 0]
-set FPGA_ID   [lindex $argv 1]
+set FPGA_PATH   [lindex $argv 1]
 set MCS       [lindex $argv 2]
 set OFFSET0   [lindex $argv 3]
 set FILE0     [lindex $argv 4]
@@ -37,8 +37,9 @@ write_cfgmem -force -format mcs -size 256 -interface SPIx4 \
 # Open and connect HW manager
 open_hw_manager
 connect_hw_server -url ${HW_SERVER} -allow_non_jtag
-current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}]
-set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}]
+
+current_hw_target [get_hw_targets *${FPGA_PATH}]
+set_property PARAM.FREQUENCY 15000000 [get_hw_targets *${FPGA_PATH}]
 open_hw_target
 current_hw_device [get_hw_devices xcvu37p_0]
 
@@ -66,11 +67,11 @@ program_hw_cfgmem -hw_cfgmem $hw_cfgmem
 
 # Program BIT
 global occ_hw_server
-global occ_target_serial
+global occ_target_path
 global occ_hw_device
 global occ_bit_stem
 set occ_hw_server $HW_SERVER
-set occ_target_serial $FPGA_ID
+set occ_target_path $FPGA_PATH
 set occ_hw_device xcvu37p_0
 set occ_bit_stem occamy_vcu128/occamy_vcu128.runs/impl_1/occamy_vcu128_wrapper
 

target/fpga/occamy_vcu128_procs.tcl

@@ -37,12 +37,12 @@ proc target_03 {} {
 
 proc occ_connect { } {
     global occ_hw_server
-    global occ_target_serial
+    global occ_target_path
     global occ_hw_device
     open_hw_manager
     connect_hw_server -url ${occ_hw_server} -allow_non_jtag
-    current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${occ_target_serial}]
-    set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${occ_target_serial}]
+    current_hw_target [get_hw_targets ${occ_target_path}]
+    set_property PARAM.FREQUENCY 15000000 [get_hw_targets ${occ_target_path}]
     open_hw_target
     current_hw_device [get_hw_devices ${occ_hw_device}]
     refresh_hw_device -update_hw_probes false [lindex [get_hw_devices ${occ_hw_device}] 0]

target/sim/sw/device/apps/libomptarget_device/src/main.c

@@ -175,6 +175,7 @@ static int gomp_offload_manager() {
 
     // (1) Wait for the offload trigger cmd == MBOX_DEVICE_START
     mailbox_read((unsigned int *)&cmd, 1);
+    cycles = read_csr(mcycle);
     if (MBOX_DEVICE_STOP == cmd) {
       //if (DEBUG_LEVEL_OFFLOAD_MANAGER > 0)
       //  snrt_trace("Got MBOX_DEVICE_STOP from host, stopping execution now.\n");
@@ -251,11 +252,8 @@ static int gomp_offload_manager() {
     // snrt_reset_perf_counter(SNRT_PERF_CNT1);
     // snrt_start_perf_counter(SNRT_PERF_CNT0, SNRT_PERF_CNT_ISSUE_FPU, core_id);
     // snrt_start_perf_counter(SNRT_PERF_CNT1, SNRT_PERF_CNT_DMA_BUSY, core_id);
-    cycles = read_csr(mcycle);
 
     offloadFn(offloadArgs);
-
-    cycles = read_csr(mcycle) - cycles;
     // snrt_stop_perf_counter(SNRT_PERF_CNT0);
     // snrt_stop_perf_counter(SNRT_PERF_CNT1);
     // issue_fpu = snrt_get_perf_counter(SNRT_PERF_CNT0);
@@ -268,6 +266,7 @@ static int gomp_offload_manager() {
     //snrt_info("cycles: %d\r\n", cycles);
 
     mailbox_write(MBOX_DEVICE_DONE);
+    cycles = read_csr(mcycle) - cycles;
     mailbox_write(cycles);
 
     //if (DEBUG_LEVEL_OFFLOAD_MANAGER > 0)

target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.c

@@ -70,13 +70,13 @@ void snrt_hero_exit(int code) { syscall(SYS_exit, code, 0, 0, 0, 0); }
 int mailbox_try_read(uint32_t *buffer) {
     return rb_device_get(g_h2a_mbox, buffer) == 0 ? 1 : 0;
 }
-int mailbox_read(uint32_t *buffer, size_t n_words) {
+inline int mailbox_read(uint32_t *buffer, size_t n_words) {
     int ret;
     while (n_words--) {
         do {
             ret = rb_device_get(g_h2a_mbox, &buffer[n_words]);
             if (ret) {
-                csleep(1000000);
+                csleep(10);
             }
         } while (ret);
     }
@@ -87,7 +87,7 @@ int mailbox_write(uint32_t word) {
     do {
         ret = rb_device_put(g_a2h_mbox, &word);
         if (ret) {
-            csleep(10000);
+            csleep(10);
         }
     } while (ret);
     return ret;

target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.h

@@ -7,6 +7,7 @@
 #include <inttypes.h>
 #include <string.h>
 #include <stdio.h>
+#include "snrt.h"
 
 /***********************************************************************************
  * MACROS
@@ -181,7 +182,12 @@ struct l3_layout {
  ***********************************************************************************/
 int syscall(uint64_t which, uint64_t arg0, uint64_t arg1, uint64_t arg2,
             uint64_t arg3, uint64_t arg4);
-void csleep(uint32_t cycles);
+
+static inline void csleep(uint32_t cycles) {
+    uint32_t start = snrt_mcycle();
+    while ((snrt_mcycle() - start) < cycles) {}
+}
+
 void snrt_hero_exit(int code);
 /**
  * @brief Blocking mailbox read access