diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5577d671 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.bender +scripts/compile.tcl + +modelsim.ini +work +transcript + diff --git a/Makefile b/Makefile index e74ca6e9..c17c5e1b 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ ROOT_DIR = $(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))) GIT ?= git BENDER ?= bender VSIM ?= vsim +VLIB ?= vlib +VOPT ?= vopt top_level ?= pulp_cluster_tb dpi-library ?= work-dpi library ?= work @@ -24,7 +26,7 @@ XVLOG_ARGS += -64bit -compile -vtimescale 1ns/1ns -quiet define generate_vsim echo 'set ROOT [file normalize [file dirname [info script]]/$3]' > $1 - bender script $(VSIM) --vlog-arg="$(VLOG_ARGS)" $2 | grep -v "set ROOT" >> $1 + bender script vsim --vlog-arg="$(VLOG_ARGS)" $2 | grep -v "set ROOT" >> $1 echo >> $1 endef @@ -76,30 +78,22 @@ sim_clean: scripts/compile.tcl: | Bender.lock $(call generate_vsim, $@, -t rtl -t test -t cluster_standalone,..) - -# compile the elfloader.cpp -$(dpi-library)/%.o: tb/dpi/%.cc $(dpi_hdr) - mkdir -p $(dpi-library) - $(CXX) -shared -fPIC -std=c++0x -Bsymbolic $(CFLAGS) -c $< -o $@ - -$(dpi-library)/cl_dpi.so: $(dpi) - $(CXX) -shared -m64 -o $(dpi-library)/cl_dpi.so $? -L$(RISCV)/lib -L$(SPIKE_ROOT)/lib -Wl,-rpath,$(RISCV)/lib -Wl,-rpath,$(SPIKE_ROOT)/lib -lfesvr + echo 'vlog "$(realpath $(ROOT_DIR))/tb/dpi/elfloader.cpp" -ccflags "-std=c++11"' >> $@ $(library): - vlib${questa_version} $(library) + $(VLIB) $(library) -compile: $(library) $(dpi) $(dpi-library)/cl_dpi.so +compile: $(library) scripts/compile.tcl @test -f Bender.lock || { echo "ERROR: Bender.lock file does not exist. Did you run make checkout in bender mode?"; exit 1; } @test -f scripts/compile.tcl || { echo "ERROR: scripts/compile.tcl file does not exist. Did you run make scripts in bender mode?"; exit 1; } - vsim -c -do 'source scripts/compile.tcl; quit' + $(VSIM) -c -do 'source scripts/compile.tcl; quit' -build: compile $(dpi) - vopt $(compile_flag) -suppress 3053 -suppress 8885 -work $(library) $(top_level) -o $(top_level)_optimized +acc -check_synthesis +build: compile + $(VOPT) $(compile_flag) -suppress 3053 -suppress 8885 -work $(library) $(top_level) -o $(top_level)_optimized +acc -check_synthesis run: - vsim +permissive $(questa-flags) $(questa-cmd) -suppress 3053 -suppress 8885 -lib $(library) +MAX_CYCLES=$(max_cycles) +UVM_TESTNAME=$(test_case) +APP=$(elf-bin) +notimingchecks +nospecify -t 1ps \ - $(uvm-flags) $(QUESTASIM_FLAGS) -sv_lib $(dpi-library)/cl_dpi \ + $(VSIM) +permissive $(questa-flags) $(questa-cmd) -suppress 3053 -suppress 8885 -lib $(library) +MAX_CYCLES=$(max_cycles) +UVM_TESTNAME=$(test_case) +APP=$(elf-bin) +notimingchecks +nospecify -t 1ps \ ${top_level}_optimized +permissive-off ++$(elf-bin) ++$(target-options) ++$(cl-bin) | tee sim.log .PHONY: test-rt-par-bare diff --git a/README.md b/README.md index e124128e..715e651d 100644 --- a/README.md +++ b/README.md @@ -26,30 +26,25 @@ To simulate the cluster on its own, you can perform the following steps: RISCV GCC toolchain](https://github.com/pulp-platform/pulp-riscv-gcc) to use a pre-built release. -2. We need RV64 toolchain to compile DPI libraries. Export it to a `RISCV` env - variable. Please refer to [RISC-V GNU - toolchain](https://github.com/riscv-collab/riscv-gnu-toolchain/) to use a - pre-built release. - -3. Compile the hw: +2. Compile the hw: ``` make checkout make scripts/compile.tcl make build ``` -4. Source the environment: - ``` - source env/env.sh - ``` - -5. Download the sw stack and bare-metal tests: +3. Download the sw stack and bare-metal tests: ``` make pulp-runtime make regression-tests ``` -6. Run the tests. Choose any test among the `parallel_bare_tests` and the +4. Source the environment: + ``` + source env/env.sh + ``` + +5. Run the tests. Choose any test among the `parallel_bare_tests` and the `mchan_tests`, move into the related folder and do: ``` diff --git a/tb/dpi/elfloader.cc b/tb/dpi/elfloader.cc deleted file mode 100644 index 2aa1dfb1..00000000 --- a/tb/dpi/elfloader.cc +++ /dev/null @@ -1,135 +0,0 @@ -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SHT_PROGBITS 0x1 -#define SHT_GROUP 0x11 - -// address and size -std::vector> sections; -std::map symbols; -// memory based address and content -std::map> mems; -reg_t entry; -int section_index = 0; - -void write (uint64_t address, uint64_t len, uint8_t* buf) { - uint64_t datum; - std::vector mem; - for (int i = 0; i < len; i++) { - mem.push_back(buf[i]); - } - mems.insert(std::make_pair(address, mem)); -} - -// Communicate the section address and len -// Returns: -// 0 if there are no more sections -// 1 if there are more sections to load -extern "C" char get_section (long long* address, long long* len) { - if (section_index < sections.size()) { - *address = sections[section_index].first; - *len = sections[section_index].second; - section_index++; - return 1; - } else return 0; -} - -extern "C" char read_section (long long address, const svOpenArrayHandle buffer) { - // get actual poitner - void* buf = svGetArrayPtr(buffer); - // check that the address points to a section - assert(mems.count(address) > 0); - // copy array - int i = 0; - for (auto &datum : mems.find(address)->second) { - *((char *) buf + i) = datum; - i++; - } -} - -extern "C" void read_elf(const char* filename) { - int fd = open(filename, O_RDONLY); - struct stat s; - assert(fd != -1); - if (fstat(fd, &s) < 0) - abort(); - size_t size = s.st_size; - - char* buf = (char*)mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - assert(buf != MAP_FAILED); - close(fd); - - assert(size >= sizeof(Elf64_Ehdr)); - const Elf64_Ehdr* eh64 = (const Elf64_Ehdr*)buf; - assert(IS_ELF32(*eh64) || IS_ELF64(*eh64)); - - - - std::vector zeros; - std::map symbols; - - #define LOAD_ELF(ehdr_t, phdr_t, shdr_t, sym_t) do { \ - ehdr_t* eh = (ehdr_t*)buf; \ - phdr_t* ph = (phdr_t*)(buf + eh->e_phoff); \ - entry = eh->e_entry; \ - assert(size >= eh->e_phoff + eh->e_phnum*sizeof(*ph)); \ - for (unsigned i = 0; i < eh->e_phnum; i++) { \ - if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { \ - if (ph[i].p_filesz) { \ - assert(size >= ph[i].p_offset + ph[i].p_filesz); \ - sections.push_back(std::make_pair(ph[i].p_paddr, ph[i].p_memsz)); \ - write(ph[i].p_paddr, ph[i].p_filesz, (uint8_t*)buf + ph[i].p_offset); \ - } \ - zeros.resize(ph[i].p_memsz - ph[i].p_filesz); \ - } \ - } \ - shdr_t* sh = (shdr_t*)(buf + eh->e_shoff); \ - assert(size >= eh->e_shoff + eh->e_shnum*sizeof(*sh)); \ - assert(eh->e_shstrndx < eh->e_shnum); \ - assert(size >= sh[eh->e_shstrndx].sh_offset + sh[eh->e_shstrndx].sh_size); \ - char *shstrtab = buf + sh[eh->e_shstrndx].sh_offset; \ - unsigned strtabidx = 0, symtabidx = 0; \ - for (unsigned i = 0; i < eh->e_shnum; i++) { \ - unsigned max_len = sh[eh->e_shstrndx].sh_size - sh[i].sh_name; \ - if ((sh[i].sh_type & SHT_GROUP) && strcmp(shstrtab + sh[i].sh_name, ".strtab") != 0 && strcmp(shstrtab + sh[i].sh_name, ".shstrtab") != 0) \ - assert(strnlen(shstrtab + sh[i].sh_name, max_len) < max_len); \ - if (sh[i].sh_type & SHT_PROGBITS) continue; \ - if (strcmp(shstrtab + sh[i].sh_name, ".strtab") == 0) \ - strtabidx = i; \ - if (strcmp(shstrtab + sh[i].sh_name, ".symtab") == 0) \ - symtabidx = i; \ - } \ - if (strtabidx && symtabidx) { \ - char* strtab = buf + sh[strtabidx].sh_offset; \ - sym_t* sym = (sym_t*)(buf + sh[symtabidx].sh_offset); \ - for (unsigned i = 0; i < sh[symtabidx].sh_size/sizeof(sym_t); i++) { \ - unsigned max_len = sh[strtabidx].sh_size - sym[i].st_name; \ - assert(sym[i].st_name < sh[strtabidx]. sh_size); \ - assert(strnlen(strtab + sym[i].st_name, max_len) < max_len); \ - symbols[strtab + sym[i].st_name] = sym[i].st_value; \ - } \ - } \ - } while(0) - - if (IS_ELF32(*eh64)) - LOAD_ELF(Elf32_Ehdr, Elf32_Phdr, Elf32_Shdr, Elf32_Sym); - else - LOAD_ELF(Elf64_Ehdr, Elf64_Phdr, Elf64_Shdr, Elf64_Sym); - - munmap(buf, size); -} diff --git a/tb/dpi/elfloader.cpp b/tb/dpi/elfloader.cpp new file mode 100644 index 00000000..757c1258 --- /dev/null +++ b/tb/dpi/elfloader.cpp @@ -0,0 +1,353 @@ +// Copyright 2022 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Modified version of the RISC-V Frontend Server +// (https://github.com/riscvarchive/riscv-fesvr, e41cfc3001293b5625c25412bd9b26e6e4ab8f7e) +// +// Nicole Narr +// Christopher Reinwardt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IS_ELF(hdr) \ + ((hdr).e_ident[0] == 0x7f && (hdr).e_ident[1] == 'E' && \ + (hdr).e_ident[2] == 'L' && (hdr).e_ident[3] == 'F') + +#define IS_ELF32(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 1) +#define IS_ELF64(hdr) (IS_ELF(hdr) && (hdr).e_ident[4] == 2) + +#define PT_LOAD 1 +#define SHT_NOBITS 8 +#define SHT_PROGBITS 0x1 +#define SHT_GROUP 0x11 + +typedef struct { + uint8_t e_ident[16]; + uint16_t e_type; + uint16_t e_machine; + uint32_t e_version; + uint32_t e_entry; + uint32_t e_phoff; + uint32_t e_shoff; + uint32_t e_flags; + uint16_t e_ehsize; + uint16_t e_phentsize; + uint16_t e_phnum; + uint16_t e_shentsize; + uint16_t e_shnum; + uint16_t e_shstrndx; +} Elf32_Ehdr; + +typedef struct { + uint32_t sh_name; + uint32_t sh_type; + uint32_t sh_flags; + uint32_t sh_addr; + uint32_t sh_offset; + uint32_t sh_size; + uint32_t sh_link; + uint32_t sh_info; + uint32_t sh_addralign; + uint32_t sh_entsize; +} Elf32_Shdr; + +typedef struct +{ + uint32_t p_type; + uint32_t p_offset; + uint32_t p_vaddr; + uint32_t p_paddr; + uint32_t p_filesz; + uint32_t p_memsz; + uint32_t p_flags; + uint32_t p_align; +} Elf32_Phdr; + +typedef struct +{ + uint32_t st_name; + uint32_t st_value; + uint32_t st_size; + uint8_t st_info; + uint8_t st_other; + uint16_t st_shndx; +} Elf32_Sym; + +typedef struct { + uint8_t e_ident[16]; + uint16_t e_type; + uint16_t e_machine; + uint32_t e_version; + uint64_t e_entry; + uint64_t e_phoff; + uint64_t e_shoff; + uint32_t e_flags; + uint16_t e_ehsize; + uint16_t e_phentsize; + uint16_t e_phnum; + uint16_t e_shentsize; + uint16_t e_shnum; + uint16_t e_shstrndx; +} Elf64_Ehdr; + +typedef struct { + uint32_t sh_name; + uint32_t sh_type; + uint64_t sh_flags; + uint64_t sh_addr; + uint64_t sh_offset; + uint64_t sh_size; + uint32_t sh_link; + uint32_t sh_info; + uint64_t sh_addralign; + uint64_t sh_entsize; +} Elf64_Shdr; + +typedef struct { + uint32_t p_type; + uint32_t p_flags; + uint64_t p_offset; + uint64_t p_vaddr; + uint64_t p_paddr; + uint64_t p_filesz; + uint64_t p_memsz; + uint64_t p_align; +} Elf64_Phdr; + +typedef struct { + uint32_t st_name; + uint8_t st_info; + uint8_t st_other; + uint16_t st_shndx; + uint64_t st_value; + uint64_t st_size; +} Elf64_Sym; + +// address and size +std::vector> sections; + +// memory based address and content +std::map> mems; + +// Entrypoint +uint64_t entry = 0; +int section_index = 0; + +extern "C" { + char get_entry(long long *entry_ret); + char get_section(long long *address_ret, long long *len_ret); + char read_section(long long address, const svOpenArrayHandle buffer, long long len); + char read_elf(const char *filename); +} + +static void write (uint64_t address, uint64_t len, uint8_t *buf) +{ + std::vector mem; + for (int i = 0; i < len; i++) { + mem.push_back(buf[i]); + } + mems.insert(std::make_pair(address, mem)); +} + +// Return the entry point reported by the ELF file +// Must be called after reading the elf file obviously +extern "C" char get_entry(long long *entry_ret) +{ + *entry_ret = entry; + return 0; +} + +// Iterator over the section addresses and lengths +// Returns: +// 0 if there are no more sections +// 1 if there are more sections to load +extern "C" char get_section(long long *address_ret, long long *len_ret) +{ + if (section_index < sections.size()) { + *address_ret = sections[section_index].first; + *len_ret = sections[section_index].second; + section_index++; + return 1; + } else { + return 0; + } +} + +extern "C" char read_section(long long address, const svOpenArrayHandle buffer, long long len) +{ + // get actual pointer + char *buf = (char *) svGetArrayPtr(buffer); + + // check that the address points to a section + if (!mems.count(address)) { + printf("[ELF] ERROR: No section found for address %p\n", address); + return -1; + } + + // copy array + long long int len_tmp = len; + for (auto &datum : mems.find(address)->second) { + if(len_tmp-- == 0){ + printf("[ELF] ERROR: Copied 0x%lx bytes. Buffer is full but there is still data available.\n", len); + return -1; + } + + *buf++ = datum; + } + + return 0; +} + +template +static void load_elf(char *buf, size_t size) +{ + E *eh = (E *) buf; + P *ph = (P *) (buf + eh->e_phoff); + Sh *sh = (Sh *) (buf + eh->e_shoff); + + char *shstrtab = NULL; + + if(size < eh->e_phoff + (eh->e_phnum * sizeof(P))){ + printf("[ELF] ERROR: Filesize is smaller than advertised program headers (0x%lx vs 0x%lx)\n", size, eh->e_phoff + (eh->e_phnum * sizeof(P))); + return; + } + + entry = eh->e_entry; + printf("[ELF] INFO: Entrypoint at %p\n", entry); + + // Iterate over all program header entries + for (unsigned int i = 0; i < eh->e_phnum; i++) { + // Check whether the current program header entry contains a loadable section of nonzero size + if(ph[i].p_type == PT_LOAD && ph[i].p_memsz) { + // Is this section something else than zeros? + if (ph[i].p_filesz) { + assert(size >= ph[i].p_offset + ph[i].p_filesz); + sections.push_back(std::make_pair(ph[i].p_paddr, ph[i].p_memsz)); + write(ph[i].p_paddr, ph[i].p_filesz, (uint8_t*)buf + ph[i].p_offset); + } + + if(ph[i].p_memsz > ph[i].p_filesz){ + printf("[ELF] WARNING: The section starting @ %p contains 0x%lx zero bytes which will NOT be preloaded!\n", + ph[i].p_paddr, (ph[i].p_memsz - ph[i].p_filesz)); + } + } + } + + if(size < eh->e_shoff + (eh->e_shnum * sizeof(Sh))){ + printf("[ELF] ERROR: Filesize is smaller than advertised section headers (0x%lx vs 0x%lx)\n", + size, eh->e_shoff + (eh->e_shnum * sizeof(Sh))); + return; + } + + if(eh->e_shstrndx >= eh->e_shnum){ + printf("[ELF] ERROR: Malformed ELF file. The index of the section header strings is out of bounds (0x%lx vs max 0x%lx)", + eh->e_shstrndx, eh->e_shnum); + return; + } + + if(size < sh[eh->e_shstrndx].sh_offset + sh[eh->e_shstrndx].sh_size){ + printf("[ELF] ERROR: Filesize is smaller than advertised size of section name table (0x%lx vs 0x%lx)\n", + size, sh[eh->e_shstrndx].sh_offset + sh[eh->e_shstrndx].sh_size); + return; + } + + // Get a direct pointer to the section name section + shstrtab = buf + sh[eh->e_shstrndx].sh_offset; + unsigned int strtabidx = 0, symtabidx = 0; + + // Iterate over all section headers to find .strtab and .symtab + for (unsigned int i = 0; i < eh->e_shnum; i++) { + // Get an upper limit on how long the name can be (length of the section name section minus the offset of the name) + unsigned int max_len = sh[eh->e_shstrndx].sh_size - sh[i].sh_name; + + // Is this the string table? + if(strcmp(shstrtab + sh[i].sh_name, ".strtab") == 0){ + printf("[ELF] INFO: Found string table at offset 0x%lx\n", sh[i].sh_offset); + strtabidx = i; + continue; + } + + // Is this the symbol table? + if(strcmp(shstrtab + sh[i].sh_name, ".symtab") == 0){ + printf("[ELF] INFO: Found symbol table at offset 0x%lx\n", sh[i].sh_offset); + symtabidx = i; + continue; + } + } +} + +extern "C" char read_elf(const char *filename) +{ + char *buf = NULL; + Elf64_Ehdr* eh64 = NULL; + int fd = open(filename, O_RDONLY); + char retval = 0; + struct stat s; + size_t size = 0; + + if(fd == -1){ + printf("[ELF] ERROR: Unable to open file %s\n", filename); + retval = -1; + goto exit; + } + + if(fstat(fd, &s) < 0) { + printf("[ELF] ERROR: Unable to read stats for file %s\n", filename); + retval = -1; + goto exit_fd; + } + + size = s.st_size; + + if(size < sizeof(Elf64_Ehdr)){ + printf("[ELF] ERROR: File %s is too small to contain a valid ELF header (0x%lx vs 0x%lx)\n", filename, size, sizeof(Elf64_Ehdr)); + retval = -1; + goto exit_fd; + } + + buf = (char *) mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); + if(buf == MAP_FAILED){ + printf("[ELF] ERROR: Unable to memory map file %s\n", filename); + retval = -1; + goto exit_fd; + } + + printf("[ELF] INFO: File %s was memory mapped to %p\n", filename, buf); + + eh64 = (Elf64_Ehdr *) buf; + + if(!(IS_ELF32(*eh64) || IS_ELF64(*eh64))){ + printf("[ELF] ERROR: File %s does not contain a valid ELF signature\n", filename); + retval = -1; + goto exit_mmap; + } + + if (IS_ELF32(*eh64)){ + load_elf(buf, size); + } else { + load_elf(buf, size); + } + +exit_mmap: + munmap(buf, size); + +exit_fd: + close(fd); + +exit: + return retval; +} \ No newline at end of file diff --git a/tb/pulp_cluster_tb.sv b/tb/pulp_cluster_tb.sv index 0ed5c3e7..6087e08a 100644 --- a/tb/pulp_cluster_tb.sv +++ b/tb/pulp_cluster_tb.sv @@ -17,13 +17,12 @@ `timescale 1ps/1ps `include "pulp_soc_defines.sv" -`include "uvm_macros.svh" `include "axi/assign.svh" `include "axi/typedef.svh" import "DPI-C" function read_elf(input string filename); import "DPI-C" function byte get_section(output longint address, output longint len); -import "DPI-C" context function byte read_section(input longint address, inout byte buffer[]); +import "DPI-C" context function byte read_section(input longint address, inout byte buffer[], input longint len); module pulp_cluster_tb; @@ -372,7 +371,7 @@ module pulp_cluster_tb; sections[section_addr >> AxiWideByteOffset] = num_words; buffer = new[num_words * AxiWideBeWidth]; - void'(read_section(section_addr, buffer)); + void'(read_section(section_addr, buffer, section_len)); for (int i = 0; i < num_words; i++) begin automatic logic [AxiWideBeWidth-1:0][7:0] word = '0; for (int j = 0; j < AxiWideBeWidth; j++) begin