Merge branch 'main' into rbarton/gemm

.github/CODEOWNERS

@@ -2,7 +2,7 @@
 # Unless a later match takes precedence, global owners below will be
 # requested for review when someone opens a pull request.
 
-* @GiannaP @colluca @paulsc96
+* @fischeti @colluca @paulsc96
 
-target/sim/sw @colluca @viv-eth
+target/sim/sw @colluca @viv-eth @fischeti
 target/fpga @niwis @CyrilKoe

.github/workflows/ci.yml

@@ -19,7 +19,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Build docs
-        run: mkdocs build
+        run: |
+          bender update && bender checkout
+          mkdocs build
 
   #######################
   # Build SW for Occamy #

.gitlab-ci.yml

@@ -10,9 +10,7 @@ variables:
   CLANG_FORMAT: clang-format-10.0.1
   CC: gcc-9.2.0
   CXX: g++-9.2.0
-  VCS: vcs-2020.12
-  VERILATOR: verilator-4.110
-  QUESTA: questa-2022.3
+  QUESTA_SEPP: questa-2022.3
   LLVM_BINROOT: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin
   CLANG: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang
   RISCV_GCC_VERSION: 8.3.0-2020.04.0
@@ -21,7 +19,7 @@ before_script:
   # yamllint disable rule:line-length
   - $PYTHON -m venv .venv
   - source .venv/bin/activate
-  - bender update
+  - $BENDER update
   - pip install -r python-requirements.txt
   # Install CVA6 compiler toolchain
   - curl -Ls -o riscv-gcc.tar.gz https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-$RISCV_GCC_VERSION-x86_64-linux-ubuntu14.tar.gz
@@ -52,10 +50,8 @@ occamy-single-cluster-vsim:
     - cd target/sim
     - make CFG_OVERRIDE=cfg/single-cluster.hjson rtl
     - make sw
-    - $QUESTA make bin/occamy_top.vsim
-    - $QUESTA ../../deps/snitch_cluster/util/sim/simulate.py
-        sw/run-single-cluster.yaml --simulator vsim
-        --sim-bin bin/occamy_top.vsim
+    - make bin/occamy_top.vsim
+    - ./run.py sw/run-single-cluster.yaml --simulator vsim
 
 #####################
 # Full Occamy tests #
@@ -67,7 +63,5 @@ occamy-full-vsim:
     - cd target/sim
     - make CFG_OVERRIDE=cfg/full.hjson rtl
     - make LENGTH=384 sw
-    - $QUESTA make bin/occamy_top.vsim
-    - $QUESTA ../../deps/snitch_cluster/util/sim/simulate.py
-        sw/run-full-occamy.yaml --simulator vsim
-        --sim-bin bin/occamy_top.vsim
+    - make bin/occamy_top.vsim
+    - ./run.py sw/run-full-occamy.yaml --simulator vsim

hw/vendor/openhwgroup_cva6/Bender.lock

@@ -22,10 +22,10 @@ packages:
       Git: https://github.com/pulp-platform/common_verification.git
     dependencies: []
   fpnew:
-    revision: 8dc44406b1ccbc4487121710c1883e805f893965
-    version: 0.6.6
+    revision: a8e0cba6dd50f357ece73c2c955d96efc3c6c315
+    version: null
     source:
-      Git: https://github.com/pulp-platform/fpnew.git
+      Git: https://github.com/pulp-platform/cvfpu.git
     dependencies:
     - common_cells
     - fpu_div_sqrt_mvp

hw/vendor/openhwgroup_cva6/Bender.yml

@@ -8,7 +8,7 @@ package:
 dependencies:
   axi: {git: https://github.com/pulp-platform/axi.git, rev: v0.39.0-beta.4}
   common_cells: {git: https://github.com/pulp-platform/common_cells.git, rev: v1.28.0}
-  fpnew: {git: https://github.com/openhwgroup/cvfpu.git, rev: 1202ca3a767b563bca5de505574373e53941506f}
+  fpnew: { git: "https://github.com/pulp-platform/cvfpu.git", rev: pulp-v0.1.3 }
   tech_cells_generic: {git: https://github.com/pulp-platform/tech_cells_generic.git, rev: v0.2.11}
 
 frozen: true

hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv

@@ -518,6 +518,7 @@ module fpu_wrap import ariane_pkg::*; (
     ) i_fpnew_bulk (
       .clk_i,
       .rst_ni,
+      .hart_id_i      ( '0                                  ),
       .operands_i     ( fpu_operands                        ),
       .rnd_mode_i     ( fpnew_pkg::roundmode_e'(fpu_rm)     ),
       .op_i           ( fpnew_pkg::operation_e'(fpu_op)     ),
@@ -527,6 +528,7 @@ module fpu_wrap import ariane_pkg::*; (
       .int_fmt_i      ( fpnew_pkg::int_format_e'(fpu_ifmt)  ),
       .vectorial_op_i ( fpu_vec_op                          ),
       .tag_i          ( fpu_tag                             ),
+      .simd_mask_i    ( '1                                  ),
       .in_valid_i     ( fpu_in_valid                        ),
       .in_ready_o     ( fpu_in_ready                        ),
       .flush_i,

hw/vendor/patches/openhwgroup_cva6/0008-cva6-Update-git-dependencies.patch

@@ -26,7 +26,7 @@ index 84b4ab8..a12628b 100644
 -    }
 +  axi: {git: https://github.com/pulp-platform/axi.git, rev: v0.39.0-beta.4}
 +  common_cells: {git: https://github.com/pulp-platform/common_cells.git, rev: v1.28.0}
-+  fpnew: {git: https://github.com/openhwgroup/cvfpu.git, rev: 1202ca3a767b563bca5de505574373e53941506f}
++  fpnew: { git: "https://github.com/pulp-platform/cvfpu.git", rev: pulp-v0.1.3 }
 +  tech_cells_generic: {git: https://github.com/pulp-platform/tech_cells_generic.git, rev: v0.2.11}
 
  frozen: true

hw/vendor/patches/openhwgroup_cva6/0009-Update-FPU-wrap-in-CVA6.patch

@@ -0,0 +1,32 @@
+From 2c8a75b2fc38f89377d4731d81e074311ed89014 Mon Sep 17 00:00:00 2001
+From: Luca Bertaccini <lbertaccini@iis.ee.ethz.ch>
+Date: Mon, 15 Jan 2024 14:29:47 +0100
+Subject: [PATCH] Update FPU wrap in CVA6
+
+---
+ hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv b/hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv
+index 1dbe526..1d5cda3 100644
+--- a/hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv
++++ b/hw/vendor/openhwgroup_cva6/core/fpu_wrap.sv
+@@ -518,6 +518,7 @@ module fpu_wrap import ariane_pkg::*; (
+     ) i_fpnew_bulk (
+       .clk_i,
+       .rst_ni,
++      .hart_id_i      ( '0                                  ),
+       .operands_i     ( fpu_operands                        ),
+       .rnd_mode_i     ( fpnew_pkg::roundmode_e'(fpu_rm)     ),
+       .op_i           ( fpnew_pkg::operation_e'(fpu_op)     ),
+@@ -527,6 +528,7 @@ module fpu_wrap import ariane_pkg::*; (
+       .int_fmt_i      ( fpnew_pkg::int_format_e'(fpu_ifmt)  ),
+       .vectorial_op_i ( fpu_vec_op                          ),
+       .tag_i          ( fpu_tag                             ),
++      .simd_mask_i    ( '1                                  ),
+       .in_valid_i     ( fpu_in_valid                        ),
+       .in_ready_o     ( fpu_in_ready                        ),
+       .flush_i,
+-- 
+2.16.5
+

target/sim/.gitignore

@@ -6,6 +6,7 @@
 /*.log
 /.*_targets_group
 /sw/**/build/
+/runs/
 
 # Auto-generated sources
 /src/

target/sim/Makefile

@@ -38,9 +38,6 @@ ADDRMAPGEN ?= $(ROOT)/util/addrmap/addrmapgen.py
 
 CLANG_FORMAT ?= $(shell which clang-format-10.0.1)
 
-VSIM = vsim
-VLOG = vlog
-
 #########################
 # Files and directories #
 #########################
@@ -523,7 +520,7 @@ clean-vlt: clean-work
 ############
 
 ${VSIM_BUILDDIR}/compile.vsim.tcl: $(VSIM_SOURCES) ${TB_SRCS} ${TB_CC_SOURCES} test/bootrom.bin | $(VSIM_BUILDDIR)
-	vlib $(dir $@)
+	$(VLIB) $(dir $@)
 	${BENDER} script vsim ${VSIM_BENDER} --vlog-arg="${VLOG_FLAGS} -work $(dir $@) " > $@
 	echo '${VLOG} -work $(dir $@) $(TB_CC_SOURCES) -ccflags "$(TB_CC_FLAGS)"' >> $@
 	echo 'return 0' >> $@

target/sim/run.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <colluca@iis.ee.ethz.ch>
+
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parent / '../../deps/snitch_cluster/util/sim'))
+from sim_utils import parser, get_simulations, run_simulations  # noqa: E402
+from Simulator import QuestaSimulator  # noqa: E402
+
+
+SIMULATORS = {
+    'vsim': QuestaSimulator(Path(__file__).parent.resolve() / 'bin/occamy_top.vsim')
+}
+
+
+def main():
+    args = parser('vsim', SIMULATORS.keys()).parse_args()
+    simulations = get_simulations(args.testlist, SIMULATORS[args.simulator])
+    return run_simulations(simulations,
+                           n_procs=args.n_procs,
+                           run_dir=Path(args.run_dir),
+                           dry_run=args.dry_run,
+                           early_exit=args.early_exit)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

target/sim/sw/device/Makefile

@@ -13,15 +13,15 @@ TARGET ?= all
 APP_SUBDIRS = $(addprefix apps/,$(APPS))
 
 SUBDIRS  = runtime
+SUBDIRS += math
 SUBDIRS += $(APP_SUBDIRS)
 
 .PHONY: all $(SUBDIRS)
 
 all: $(SUBDIRS)
 
-runtime:
+$(SUBDIRS):
 	$(MAKE) -C $@ $(TARGET)
 
 # Explicit dependency of apps on runtime
-$(APP_SUBDIRS): runtime
-	$(MAKE) -C $@ $(TARGET)
+$(APP_SUBDIRS): runtime math

target/sim/sw/device/apps/common.mk

@@ -9,17 +9,25 @@
 MK_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
 include $(MK_DIR)/../toolchain.mk
 
-###################
-# Build variables #
-###################
+###############
+# Directories #
+###############
 
-# Directories
-BUILDDIR    = $(abspath build)
-APPSDIR     = $(abspath $(MK_DIR))
-RUNTIME_DIR = $(abspath $(MK_DIR)/../runtime)
+# Fixed paths in repository tree
+ROOT        = $(abspath $(MK_DIR)/../../../../../)
 SNITCH_ROOT = $(shell bender path snitch_cluster)
+APPSDIR     = $(abspath $(MK_DIR))
+RUNTIME_DIR = $(ROOT)/target/sim/sw/device/runtime
 SNRT_DIR    = $(SNITCH_ROOT)/sw/snRuntime
-SW_DIR      = $(abspath $(MK_DIR)/../../)
+SW_DIR      = $(ROOT)/target/sim/sw/
+MATH_DIR    = $(ROOT)/target/sim/sw/device/math
+
+# Paths relative to the app including this Makefile
+BUILDDIR    = $(abspath build)
+
+###################
+# Build variables #
+###################
 
 # Dependencies
 INCDIRS += $(RUNTIME_DIR)/src
@@ -29,14 +37,7 @@ INCDIRS += $(SNRT_DIR)/vendor/riscv-opcodes
 INCDIRS += $(SW_DIR)/shared/platform/generated
 INCDIRS += $(SW_DIR)/shared/platform
 INCDIRS += $(SW_DIR)/shared/runtime
-
-# Math library override
-INCDIRS += $(SNITCH_ROOT)/sw/math/arch/riscv64/bits/
-INCDIRS += $(SNITCH_ROOT)/sw/math/arch/generic
-INCDIRS += $(SNITCH_ROOT)/sw/math/src/include
-INCDIRS += $(SNITCH_ROOT)/sw/math/src/internal
-INCDIRS += $(SNITCH_ROOT)/sw/math/include/bits
-INCDIRS += $(SNITCH_ROOT)/sw/math/include
+INCDIRS += $(SNITCH_ROOT)/sw/blas
 
 # Linking sources
 BASE_LD       = $(abspath $(SNRT_DIR)/base.ld)
@@ -55,6 +56,10 @@ RISCV_LDFLAGS += -T$(BASE_LD)
 # Link snRuntime library
 RISCV_LDFLAGS += -L$(SNRT_LIB_DIR)
 RISCV_LDFLAGS += -l$(SNRT_LIB_NAME)
+# Link math library
+RISCV_LDFLAGS += -L$(MATH_DIR)/build
+RISCV_LDFLAGS += -lmath
+
 
 # Objcopy flags
 OBJCOPY_FLAGS  = -O binary

target/sim/sw/device/math/Makefile

@@ -0,0 +1,8 @@
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <colluca@iis.ee.ethz.ch>
+
+include ../toolchain.mk
+include $(SNITCH_ROOT)/sw/math/Makefile

target/sim/sw/device/runtime/Makefile

@@ -44,7 +44,7 @@ ALL_OUTPUTS = $(DEPS) $(LIB) $(DUMP)
 #########
 
 .PHONY: all
-all: math $(ALL_OUTPUTS)
+all: $(ALL_OUTPUTS)
 
 .PHONY: clean
 clean:
@@ -68,11 +68,6 @@ $(LIB): $(OBJS) | $(BUILDDIR)
 $(DUMP): $(LIB) | $(BUILDDIR)
 	$(RISCV_OBJDUMP) -D $< > $@
 
-# Build math library sources
-.PHONY: math
-math:
-	make -C $(SNITCH_ROOT)/sw/math all
-
 ifneq ($(MAKECMDGOALS),clean)
 -include $(DEPS)
 endif

target/sim/sw/device/runtime/src/snrt.h

@@ -27,6 +27,7 @@
 #include "cls.h"
 #include "cluster_interrupts.h"
 #include "dma.h"
+#include "dump.h"
 #include "global_interrupts.h"
 #include "occamy_device.h"
 #include "occamy_memory.h"

target/sim/sw/device/toolchain.mk

@@ -4,4 +4,6 @@
 #
 # Luca Colagrande <colluca@iis.ee.ethz.ch>
 
-include $(shell bender path snitch_cluster)/target/snitch_cluster/sw/toolchain.mk
+BENDER ?= bender
+SNITCH_ROOT = $(shell $(BENDER) path snitch_cluster)
+include $(SNITCH_ROOT)/target/snitch_cluster/sw/toolchain.mk

target/sim/sw/host/apps/hello_world/verify.py

@@ -10,19 +10,16 @@
 import sys
 
 sys.path.append(str(Path(__file__).parent / '../../../../../../deps/snitch_cluster/util/sim/'))
-from simulate import run_simulation  # noqa: E402
+from sim_utils import run_simulations  # noqa: E402
+from Simulator import QuestaSimulator  # noqa: E402
 
-UART_LOG = str(Path(__file__).parent / '../../../../uart0.log')
+UART_LOG = 'uart0.log'
 EXPECTED_OUTPUT = "Hello world!\r\n"
 
 
 def parse_args():
     # Argument parsing
     parser = argparse.ArgumentParser(allow_abbrev=True)
-    parser.add_argument(
-        'simulator',
-        help='The simulator to be used',
-    )
     parser.add_argument(
         'sim_bin',
         help='The simulator binary to be used to start the simulation',
@@ -35,8 +32,10 @@ def parse_args():
 
 def main():
     args = parse_args()
-    cmd = f"{args.sim_bin} {args.snitch_bin}"
-    result, _ = run_simulation(cmd, args.simulator, {})
+    simulator = QuestaSimulator(args.sim_bin)
+    simulation = simulator.get_simulation({'elf': args.snitch_bin})
+    result = run_simulations([simulation])
+
     actual_output = ''
     with open(UART_LOG, 'rb') as file:
         actual_output = file.read().decode('ascii')

target/sim/sw/run-full-occamy.yaml

@@ -4,8 +4,8 @@
 
 runs:
   - elf: host/apps/offload/build/offload-axpy.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/axpy/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/axpy/build/axpy.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/axpy/verify.py, --symbols-bin,
+          ./device/apps/blas/axpy/build/axpy.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/offload/build/offload-gemm.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/gemm/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/gemm/build/gemm.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/gemm/verify.py, --symbols-bin,
+          ./device/apps/blas/gemm/build/gemm.elf, "${sim_bin}", "${elf}"]

target/sim/sw/run-single-cluster.yaml

@@ -4,10 +4,10 @@
 
 runs:
   - elf: host/apps/offload/build/offload-axpy.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/axpy/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/axpy/build/axpy.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/axpy/verify.py, --symbols-bin,
+          ./device/apps/blas/axpy/build/axpy.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/offload/build/offload-gemm.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/gemm/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/gemm/build/gemm.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/gemm/verify.py, --symbols-bin,
+          ./device/apps/blas/gemm/build/gemm.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/hello_world/build/hello_world.elf
-    cmd: sw/host/apps/hello_world/verify.py {simulator} {sim_bin} {elf}
+    cmd: [./host/apps/hello_world/verify.py, "${sim_bin}", "${elf}"]