diff --git a/target/fpga/Makefile b/target/fpga/Makefile index 47733aa7a..b5addbb0f 100644 --- a/target/fpga/Makefile +++ b/target/fpga/Makefile @@ -7,12 +7,11 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) MKFILE_DIR := $(dir $(MKFILE_PATH)) ROOT := ${MKFILE_DIR}../../../.. -CVA6_SDK ?= ${ROOT}/../cva6-sdk +CVA6_SDK ?= DEBUG ?= 0 EXT_JTAG ?= 0 -VCU ?= 01 -FPGA_ID := -HW_SERVER := +FPGA_PATH := $(XILINX_FPGA_PATH) +HW_SERVER := $(XILINX_HOST):$(XILINX_PORT) BENDER ?= bender VIVADO ?= vitis-2020.2 vivado # Do not proceed with implem (CI) @@ -54,12 +53,12 @@ program: ${VIVADO} -mode batch -source occamy_vcu128_program.tcl -tclargs ${VCU} flash: ${FILE} - ${VIVADO} -mode batch -source occamy_vcu128_flash.tcl -tclargs ${HW_SERVER} ${FPGA_ID} flash.mcs ${OFFSET} ${FILE} + ${VIVADO} -mode batch -source occamy_vcu128_flash.tcl -tclargs ${HW_SERVER} ${FPGA_PATH} flash.mcs ${OFFSET} ${FILE} rm flash.mcs # Flash only uboot (made for TFTP bootmode), overwrite the bootrom, and reset the board flashrun: ${UBOOT_ITB} - ${VIVADO} -mode batch -source occamy_vcu128_flashrun.tcl -tclargs ${HW_SERVER} ${FPGA_ID} flash.mcs 6000000 ${UBOOT_ITB} + ${VIVADO} -mode batch -source occamy_vcu128_flashrun.tcl -tclargs ${HW_SERVER} ${FPGA_PATH} flash.mcs 6000000 ${UBOOT_ITB} rm flash.mcs flash-u-boot: diff --git a/target/fpga/bootrom/Makefile b/target/fpga/bootrom/Makefile index d156130cb..ff8cbe736 100644 --- a/target/fpga/bootrom/Makefile +++ b/target/fpga/bootrom/Makefile @@ -38,8 +38,8 @@ all: bootrom-spl.coe bootrom.tcl bootrom-spl.tcl $(CC) $(CFLAGS) -DGIT_SHA=\"$(GIT_SHA)\" $(INCLUDES) -c $< -o $@ @echo "CC <= $<" -%.dtb: %.dts - dtc -I dts $< -O dtb -o $@ +%.dtb: %.dts $(wildcard *.dtsi) + dtc -@ -I dts $< -O dtb -o $@ bootrom.elf bootrom.dump bootrom.bin: bootrom.S $(OBJS_C) bootrom.ld occamy.dtb $(CC) $(CFLAGS) $(INCLUDES) -Tbootrom.ld $< $(OBJS_C) -o bootrom.elf diff --git a/target/fpga/bootrom/mac_address.dtsi b/target/fpga/bootrom/mac_address.dtsi new file mode 100644 index 000000000..8ad365a59 --- /dev/null +++ b/target/fpga/bootrom/mac_address.dtsi @@ -0,0 +1,2 @@ +local-mac-address = [ 00 00 00 00 00 00 ]; +mac-address = [ 00 00 00 00 00 00 ]; diff --git a/target/fpga/bootrom/occamy.dts b/target/fpga/bootrom/occamy.dts index dcd2d3954..0e8e785a2 100644 --- a/target/fpga/bootrom/occamy.dts +++ b/target/fpga/bootrom/occamy.dts @@ -191,8 +191,8 @@ clock-names = "s_axi_lite_clk", "axis_clk"; // interrupt and mac_irq interrupts-extended = <&PLIC0 1 &PLIC0 6>; - local-mac-address = [ 00 0A 35 07 D5 DD ]; // hero-vcu128-03 - mac-address = [ 00 0A 35 07 D5 DD ]; // hero-vcu128-03 + /include/ "mac_address.dtsi" + /include/ "remote_boot.dtsi" device_type = "network"; axistream-connected = <ð_dma0>; axistream-control-connected = <ð_dma0>; diff --git a/target/fpga/bootrom/occamy_pcie.dts b/target/fpga/bootrom/occamy_pcie.dts index 0ccfcf4ec..6873bf14d 100644 --- a/target/fpga/bootrom/occamy_pcie.dts +++ b/target/fpga/bootrom/occamy_pcie.dts @@ -117,9 +117,15 @@ reg-names = "quadrant-control"; reg = <0x0 0x0b000000 0x0 0x10000>; }; - scratchpad-narrow@70000000 { - compatible = "eth,scratchpad-narrow"; - reg = <0x0 0x70000000 0x0 0x80000>; + // We do not use the spm-narrow (contains OpenSBI code) + spm_wide: spm-wide@71000000 { + compatible = "eth,occamy-spm-wide"; + reg-names = "spm-wide"; + reg = <0x0 0x71000000 0x0 0x100000>; + }; + pcie_axi_bar_mem: pcie-axi-bar-mem@20000000 { + compatible = "eth,pcie-axi-bar-mem"; + reg = <0x0 0x20000000 0x0 0x40000000>; }; // Instantiate a snitch cluster snitch-cluster@10000000 { diff --git a/target/fpga/bootrom/remote_boot.dtsi b/target/fpga/bootrom/remote_boot.dtsi new file mode 100644 index 000000000..e69de29bb diff --git a/target/fpga/bootrom/src/main.c b/target/fpga/bootrom/src/main.c index 92da79812..987d07035 100644 --- a/target/fpga/bootrom/src/main.c +++ b/target/fpga/bootrom/src/main.c @@ -79,7 +79,7 @@ int main() { // Copy the DBT at (SPM+4) print_uart("\r\nCopying DTB at "); - print_uart_addr(SPL_DEST + 1); + print_uart_addr(SPL_DEST + 4); for (int i = 0; i < totalsize; i++) *(uint8_t *)(SPL_DEST + 4 + i) = *(((uint8_t *)__dtb_start) + i); diff --git a/target/fpga/occamy_vcu128_bd.tcl b/target/fpga/occamy_vcu128_bd.tcl index 7d66e6c18..77046e896 100644 --- a/target/fpga/occamy_vcu128_bd.tcl +++ b/target/fpga/occamy_vcu128_bd.tcl @@ -710,8 +710,15 @@ proc create_root_design { parentCell } { CONFIG.axist_bypass_scale {Gigabytes} \ CONFIG.axist_bypass_size {4} \ CONFIG.axisten_freq {125} \ + CONFIG.bar_indicator {BAR_1:0} \ CONFIG.functional_mode {AXI_Bridge} \ + CONFIG.pf0_bar0_64bit {true} \ + CONFIG.pf0_bar0_prefetchable {true} \ + CONFIG.pf0_bar0_scale {Gigabytes} \ + CONFIG.pf0_bar0_size {4} \ CONFIG.pf0_device_id {9014} \ + CONFIG.pf0_msix_cap_pba_bir {BAR_1:0} \ + CONFIG.pf0_msix_cap_table_bir {BAR_1:0} \ CONFIG.pl_link_cap_max_link_width {X4} \ CONFIG.xdma_axi_intf_mm {AXI_Memory_Mapped} \ CONFIG.xdma_axilite_slave {true} \ @@ -874,7 +881,7 @@ proc create_root_design { parentCell } { assign_bd_address -offset 0x4CC00000 -range 0x00400000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs hbm_0/SAPB_0/Reg] -force assign_bd_address -offset 0x4C800000 -range 0x00400000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs hbm_0/SAPB_1/Reg] -force assign_bd_address -offset 0x20000000 -range 0x20000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_B/BAR0] -force - assign_bd_address -offset 0x00000000 -range 0x20000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_LITE/CTL0] -force + assign_bd_address -offset 0x4E000000 -range 0x01000000 -target_address_space [get_bd_addr_spaces occamy/m_axi_pcie] [get_bd_addr_segs xdma_0/S_AXI_LITE/CTL0] -force assign_bd_address -offset 0x00000000 -range 0x0001000000000000 -target_address_space [get_bd_addr_spaces xdma_0/M_AXI_B] [get_bd_addr_segs occamy/s_axi_pcie/reg0] -force # Exclude Address Segments diff --git a/target/fpga/occamy_vcu128_flash.tcl b/target/fpga/occamy_vcu128_flash.tcl index 22c3e5964..c3efb6fab 100644 --- a/target/fpga/occamy_vcu128_flash.tcl +++ b/target/fpga/occamy_vcu128_flash.tcl @@ -8,17 +8,17 @@ # Programs the SPI Flash of the VCU128 board with with two partitions # # HW_SERVER host:port URL to the server where the FPGA board is connected to -# FPGA_ID Serial of the FPGA to target +# FPGA_PATH Serial of the FPGA to target # MCS Output flash configuration file # OFFSET0 Address offset of partition 0 # FILE0 File to program to partition 0 # Parse arguments if {$argc < 5} { - error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_ID MCS OFFSET0 FILE0" + error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_PATH MCS OFFSET0 FILE0" } set HW_SERVER [lindex $argv 0] -set FPGA_ID [lindex $argv 1] +set FPGA_PATH [lindex $argv 1] set MCS [lindex $argv 2] set OFFSET0 [lindex $argv 3] set FILE0 [lindex $argv 4] @@ -38,8 +38,8 @@ write_cfgmem -force -format mcs -size 256 -interface SPIx4 \ # Open and connect HW manager open_hw_manager connect_hw_server -url ${HW_SERVER} -allow_non_jtag -current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}] -set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}] +current_hw_target [get_hw_targets ${FPGA_PATH}] +set_property PARAM.FREQUENCY 15000000 [get_hw_targets ${FPGA_PATH}] open_hw_target current_hw_device [get_hw_devices xcvu37p_0] diff --git a/target/fpga/occamy_vcu128_flashrun.tcl b/target/fpga/occamy_vcu128_flashrun.tcl index 9f9d59508..c92c925e9 100644 --- a/target/fpga/occamy_vcu128_flashrun.tcl +++ b/target/fpga/occamy_vcu128_flashrun.tcl @@ -9,7 +9,7 @@ # Afterwards programs the real bitstream and runs the bootrom # # HW_SERVER host:port URL to the server where the FPGA board is connected to -# FPGA_ID Serial of the FPGA to target +# FPGA_PATH Serial of the FPGA to target # MCS Output flash configuration file # OFFSET0 Address offset of partition 0 # FILE0 File to program to partition 0 @@ -18,10 +18,10 @@ source occamy_vcu128_procs.tcl # Parse arguments if {$argc < 5} { - error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_ID MCS OFFSET0 FILE0" + error "usage: occamy_vcu_138_flash.tcl HW_SERVER FPGA_PATH MCS OFFSET0 FILE0" } set HW_SERVER [lindex $argv 0] -set FPGA_ID [lindex $argv 1] +set FPGA_PATH [lindex $argv 1] set MCS [lindex $argv 2] set OFFSET0 [lindex $argv 3] set FILE0 [lindex $argv 4] @@ -37,8 +37,9 @@ write_cfgmem -force -format mcs -size 256 -interface SPIx4 \ # Open and connect HW manager open_hw_manager connect_hw_server -url ${HW_SERVER} -allow_non_jtag -current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}] -set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${FPGA_ID}] + +current_hw_target [get_hw_targets *${FPGA_PATH}] +set_property PARAM.FREQUENCY 15000000 [get_hw_targets *${FPGA_PATH}] open_hw_target current_hw_device [get_hw_devices xcvu37p_0] @@ -66,11 +67,11 @@ program_hw_cfgmem -hw_cfgmem $hw_cfgmem # Program BIT global occ_hw_server -global occ_target_serial +global occ_target_path global occ_hw_device global occ_bit_stem set occ_hw_server $HW_SERVER -set occ_target_serial $FPGA_ID +set occ_target_path $FPGA_PATH set occ_hw_device xcvu37p_0 set occ_bit_stem occamy_vcu128/occamy_vcu128.runs/impl_1/occamy_vcu128_wrapper diff --git a/target/fpga/occamy_vcu128_procs.tcl b/target/fpga/occamy_vcu128_procs.tcl index 6e2054c39..12c9ede1a 100644 --- a/target/fpga/occamy_vcu128_procs.tcl +++ b/target/fpga/occamy_vcu128_procs.tcl @@ -37,12 +37,12 @@ proc target_03 {} { proc occ_connect { } { global occ_hw_server - global occ_target_serial + global occ_target_path global occ_hw_device open_hw_manager connect_hw_server -url ${occ_hw_server} -allow_non_jtag - current_hw_target [get_hw_targets */xilinx_tcf/Xilinx/${occ_target_serial}] - set_property PARAM.FREQUENCY 15000000 [get_hw_targets */xilinx_tcf/Xilinx/${occ_target_serial}] + current_hw_target [get_hw_targets ${occ_target_path}] + set_property PARAM.FREQUENCY 15000000 [get_hw_targets ${occ_target_path}] open_hw_target current_hw_device [get_hw_devices ${occ_hw_device}] refresh_hw_device -update_hw_probes false [lindex [get_hw_devices ${occ_hw_device}] 0] diff --git a/target/sim/sw/device/apps/libomptarget_device/src/main.c b/target/sim/sw/device/apps/libomptarget_device/src/main.c index dc28136fa..ca1a4c7b1 100644 --- a/target/sim/sw/device/apps/libomptarget_device/src/main.c +++ b/target/sim/sw/device/apps/libomptarget_device/src/main.c @@ -175,6 +175,7 @@ static int gomp_offload_manager() { // (1) Wait for the offload trigger cmd == MBOX_DEVICE_START mailbox_read((unsigned int *)&cmd, 1); + cycles = read_csr(mcycle); if (MBOX_DEVICE_STOP == cmd) { //if (DEBUG_LEVEL_OFFLOAD_MANAGER > 0) // snrt_trace("Got MBOX_DEVICE_STOP from host, stopping execution now.\n"); @@ -251,11 +252,8 @@ static int gomp_offload_manager() { // snrt_reset_perf_counter(SNRT_PERF_CNT1); // snrt_start_perf_counter(SNRT_PERF_CNT0, SNRT_PERF_CNT_ISSUE_FPU, core_id); // snrt_start_perf_counter(SNRT_PERF_CNT1, SNRT_PERF_CNT_DMA_BUSY, core_id); - cycles = read_csr(mcycle); offloadFn(offloadArgs); - - cycles = read_csr(mcycle) - cycles; // snrt_stop_perf_counter(SNRT_PERF_CNT0); // snrt_stop_perf_counter(SNRT_PERF_CNT1); // issue_fpu = snrt_get_perf_counter(SNRT_PERF_CNT0); @@ -268,6 +266,7 @@ static int gomp_offload_manager() { //snrt_info("cycles: %d\r\n", cycles); mailbox_write(MBOX_DEVICE_DONE); + cycles = read_csr(mcycle) - cycles; mailbox_write(cycles); //if (DEBUG_LEVEL_OFFLOAD_MANAGER > 0) diff --git a/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.c b/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.c index 4ebfe486e..f6e276a0c 100644 --- a/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.c +++ b/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.c @@ -70,13 +70,13 @@ void snrt_hero_exit(int code) { syscall(SYS_exit, code, 0, 0, 0, 0); } int mailbox_try_read(uint32_t *buffer) { return rb_device_get(g_h2a_mbox, buffer) == 0 ? 1 : 0; } -int mailbox_read(uint32_t *buffer, size_t n_words) { +inline int mailbox_read(uint32_t *buffer, size_t n_words) { int ret; while (n_words--) { do { ret = rb_device_get(g_h2a_mbox, &buffer[n_words]); if (ret) { - csleep(1000000); + csleep(10); } } while (ret); } @@ -87,7 +87,7 @@ int mailbox_write(uint32_t word) { do { ret = rb_device_put(g_a2h_mbox, &word); if (ret) { - csleep(10000); + csleep(10); } } while (ret); return ret; diff --git a/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.h b/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.h index 8d532e46e..eef409566 100644 --- a/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.h +++ b/target/sim/sw/device/apps/libomptarget_device/src/sw_mailbox.h @@ -7,6 +7,7 @@ #include #include #include +#include "snrt.h" /*********************************************************************************** * MACROS @@ -181,7 +182,12 @@ struct l3_layout { ***********************************************************************************/ int syscall(uint64_t which, uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4); -void csleep(uint32_t cycles); + +static inline void csleep(uint32_t cycles) { + uint32_t start = snrt_mcycle(); + while ((snrt_mcycle() - start) < cycles) {} +} + void snrt_hero_exit(int code); /** * @brief Blocking mailbox read access