From 879f37e4f44ec4e4645c48b297f9c815b70a379a Mon Sep 17 00:00:00 2001 From: Maximilian Date: Wed, 27 Nov 2024 09:56:46 +0100 Subject: [PATCH] DPI implemented - connections in the RoCE-stack still missing --- driver/coyote_dev.h | 3 + driver/fpga_mmu.c | 16 ++ examples_hw/CMakeLists.txt | 25 +++ examples_hw/apps/rdma_dpi/init_ip.tcl | 0 examples_hw/apps/rdma_dpi/vfpga_top.svh | 118 +++++++++++ .../rdma_service_no_daemon/client/main.cpp | 2 +- .../rdma_service_no_daemon/server/main.cpp | 2 +- .../network/rdma/dpi_transmission_dropper.sv | 185 ++++++++++++++++++ .../intrusion_detection_data_aggregator.sv | 4 + .../rdma/intrusion_detection_decider.sv | 5 +- hw/hdl/network/rdma/payload_extractor.sv | 6 +- hw/hdl/network/rdma/roce_stack.sv | 6 +- .../wr_hdl/template_gen/lynx_pkg_tmplt.txt | 10 + sw/include/bThread.hpp | 1 + sw/include/cLib.hpp | 6 + sw/src/bThread.cpp | 28 +++ 16 files changed, 411 insertions(+), 6 deletions(-) create mode 100644 examples_hw/apps/rdma_dpi/init_ip.tcl create mode 100644 examples_hw/apps/rdma_dpi/vfpga_top.svh create mode 100644 hw/hdl/network/rdma/dpi_transmission_dropper.sv diff --git a/driver/coyote_dev.h b/driver/coyote_dev.h index ced1864e..314ceb46 100644 --- a/driver/coyote_dev.h +++ b/driver/coyote_dev.h @@ -367,6 +367,9 @@ extern char *config_fname; #define IRQ_NOTIFY 4 #define IRQ_RCNFG 5 +// Enter a new IRQ for DPI-flagging +#define IRQ_DPI 6 + /** * @brief Cdev * diff --git a/driver/fpga_mmu.c b/driver/fpga_mmu.c index 36908d5b..a7f8e11e 100644 --- a/driver/fpga_mmu.c +++ b/driver/fpga_mmu.c @@ -111,6 +111,22 @@ irqreturn_t fpga_isr(int irq, void *dev_id) pr_err("could not enqueue a workqueue, notify ISR"); } break; + + case IRQ_DPI: + dbg_info("(irq=%d) notify, vFPGA %d\n", irq, d->id); + dbg_info("Packet was flagged by DPI and transmission was stopped. Application will be stopped now.") + irq_not = kzalloc(sizeof(struct fpga_irq_notify), GFP_KERNEL); + BUG_ON(!irq_not); + + irq_not->d = d; + fpga_read_irq_notify(d, irq_not); + + INIT_WORK(&irq_not->work_notify, fpga_notify_handler); + + if(!queue_work(d->wqueue_notify, &irq_not->work_notify)) { + pr_err("could not enqueue a workqueue, notify ISR"); + } + break; default: break; diff --git a/examples_hw/CMakeLists.txt b/examples_hw/CMakeLists.txt index fb4c7d80..1ff6ebd7 100644 --- a/examples_hw/CMakeLists.txt +++ b/examples_hw/CMakeLists.txt @@ -183,6 +183,11 @@ if(EXAMPLE STREQUAL "rdma_aes") create_hw() endif() +# Network example, RDMA +# @brief: RDMA for microbenchmarks +# +# @note: Add a target device (FDEV_NAME) +# if(EXAMPLE STREQUAL "rdma_perf") message("** RDMA perf test") set(SHELL_PROBE 6) @@ -200,6 +205,26 @@ if(EXAMPLE STREQUAL "rdma_perf") create_hw() endif() +# Network example, RDMA +# @brief: RDMA with DPI +# +# @note: Add a target device (FDEV_NAME) +# +if(EXAMPLE STREQUAL "rdma_dpi") + message("** RDMA DPI test") + set(SHELL_PROBE 8) + set(N_REGIONS 1) + set(EN_STRM 1) + set(EN_RDMA 1) + set(N_STRM_AXI 2) + + validation_checks_hw() + + load_apps ( + VFPGA_C0_0 "apps/rdma_dpi" + ) +endif() + # Network example, TCP iperf # @brief: TCP iperf # diff --git a/examples_hw/apps/rdma_dpi/init_ip.tcl b/examples_hw/apps/rdma_dpi/init_ip.tcl new file mode 100644 index 00000000..e69de29b diff --git a/examples_hw/apps/rdma_dpi/vfpga_top.svh b/examples_hw/apps/rdma_dpi/vfpga_top.svh new file mode 100644 index 00000000..f0843024 --- /dev/null +++ b/examples_hw/apps/rdma_dpi/vfpga_top.svh @@ -0,0 +1,118 @@ +/** + * VFPGA TOP FOR DPI + * + * Catch write commands to check if an IRQ needs to be raised or not + * + */ + +// Direct comb of the read-interface, we're only looking at the write-commands for IRQs +always_comb begin + // Read ops + sq_rd.valid = rq_rd.valid; + rq_rd.ready = sq_rd.ready; + sq_rd.data = rq_rd.data; + // OW + sq_rd.data.strm = STRM_HOST; + sq_rd.data.dest = 1; +end + +// Introduce additional pipeline stage that allows to modify the write commands if it's actually malicious +req_t sq_wr_data_intermediate; +logic sq_wr_valid_intermediate; + +irq_not_t notify_data_intermediate; +logic notify_valid_intermediate; + +logic interrupt_raised; + +// Do the required assignments to the intermediate pipeline-stage +assign sq_wr.valid = sq_wr_valid_intermediate; +assign sq_wr.data = sq_wr_data_intermediate; +assign rq_rd.ready = sq_wr.ready; + +assign notify.valid = notify_valid_intermediate; +assign notify.data = notify_data_intermediate; + +// Synchronous block to catch incoming commands and generate IRQs if required +always_ff @(posedge aclk) begin + if(!aresetn) begin + // Reset all intermediate registers + sq_wr_data_intermediate <= 512'b0; + sq_wr_valid_intermediate <= 1'b0; + + notify_valid_intermediate <= 1'b0; + notify_data_intermediate <= 38'b0; + + // Reset the interrupt-raised register + interrupt_raised <= 1'b0; + end else begin + // Wait for incoming command + if(rq_wr.valid && sq_wr.ready) begin + // Check the opcode of this command + if(rq_wr.data.opcode == RC_ROCE_DPI_IRQ) begin + // If the incoming command-opcode is the specialized DPI_IRQ, stop the transmission of the command and raise an actual interrupt + sq_wr_data_intermediate <= 512'b0; + sq_wr_valid_intermediate <= 1'b0; + + // For the interrupt, forward the pid from the incoming command + notify_valid_intermediate <= 1'b1; + notify_data_intermediate.pid <= rq_wr.data.pid; + notify_data_intermediate.value <= IRQ_DPI; + + // Set the bit that indicates that the interrupt was written + interrupt_raised <= 1; + end else begin + + end + end else begin + // If there's no command coming in, just reset the intermediate command stage + sq_wr_data_intermediate <= 512'b0; + sq_wr_valid_intermediate <= 1'b0; + end + + // Check if interrupt was raised and picked up (ready-signal!) If so, it can be deasserted. + if(interrupt_raised && notify.ready) begin + interrupt_raised <= 1'b0; + notify_data_intermediate <= 38'b0; + notify_valid_intermediate <= 1'b0; + end + end +end + +`AXISR_ASSIGN(axis_host_recv[0], axis_rreq_send[0]) +`AXISR_ASSIGN(axis_rreq_recv[0], axis_host_send[0]) +`AXISR_ASSIGN(axis_host_recv[1], axis_rrsp_send[0]) +`AXISR_ASSIGN(axis_rrsp_recv[0], axis_host_send[1]) + +ila_0 inst_ila ( + .clk(aclk), + .probe0(axis_host_recv[0].tvalid), + .probe1(axis_host_recv[0].tready), + .probe2(axis_host_recv[0].tlast), + + .probe3(axis_host_recv[1].tvalid), + .probe4(axis_host_recv[1].tready), + .probe5(axis_host_recv[1].tlast), + + .probe6(axis_host_send[0].tvalid), + .probe7(axis_host_send[0].tready), + .probe8(axis_host_send[0].tlast), + + .probe9(axis_host_send[1].tvalid), + .probe10(axis_host_send[1].tready), + .probe11(axis_host_send[1].tlast), + + .probe12(sq_wr.valid), + .probe13(sq_wr.ready), + .probe14(sq_wr.data), // 128 + .probe15(sq_rd.valid), + .probe16(sq_rd.ready), + .probe17(sq_rd.data), // 128 + .probe18(cq_rd.valid), + .probe19(cq_wr.valid) +); + +// Tie-off unused +always_comb axi_ctrl.tie_off_s(); +always_comb cq_rd.tie_off_s(); +always_comb cq_wr.tie_off_s(); \ No newline at end of file diff --git a/examples_sw/apps/rdma_service_no_daemon/client/main.cpp b/examples_sw/apps/rdma_service_no_daemon/client/main.cpp index 0c9d7a9b..338669c9 100644 --- a/examples_sw/apps/rdma_service_no_daemon/client/main.cpp +++ b/examples_sw/apps/rdma_service_no_daemon/client/main.cpp @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) ("verbose,v", boost::program_options::value(), "Printout of single messages") ("encryption,e", boost::program_options::value(), "Encryption required") ("compression,c", boost::program_options::value(), "Compression required") - ("dpi,d", boost::program_options::value(), "DPI required"); + ("dpi,p", boost::program_options::value(), "DPI required"); boost::program_options::variables_map commandLineArgs; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, programDescription), commandLineArgs); diff --git a/examples_sw/apps/rdma_service_no_daemon/server/main.cpp b/examples_sw/apps/rdma_service_no_daemon/server/main.cpp index 062c6cad..9f3632ea 100644 --- a/examples_sw/apps/rdma_service_no_daemon/server/main.cpp +++ b/examples_sw/apps/rdma_service_no_daemon/server/main.cpp @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) ("verbose,v", boost::program_options::value(), "Printout of single messages") ("encryption,e", boost::program_options::value(), "Encryption required") ("compression,c", boost::program_options::value(), "Compression required") - ("dpi,d", boost::program_options::value(), "DPI required"); + ("dpi,p", boost::program_options::value(), "DPI required"); boost::program_options::variables_map commandLineArgs; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, programDescription), commandLineArgs); diff --git a/hw/hdl/network/rdma/dpi_transmission_dropper.sv b/hw/hdl/network/rdma/dpi_transmission_dropper.sv new file mode 100644 index 00000000..ebb0ac43 --- /dev/null +++ b/hw/hdl/network/rdma/dpi_transmission_dropper.sv @@ -0,0 +1,185 @@ +// Module that sits on the output data stream from the HLS-stack, reads incoming data and DPI-decisions and can then either drop the payload or not + + +// Import the lynxTypes to be able to reference the datatypes +import lynxTypes::*; + +module dpi_transmission_dropper( + // Incoming clock and reset signal + input logic nclk, + input logic nresetn, + + // Incoming data stream from the HLS-stack + AXI4S.s s_axis_rdma_wr, + + // Outgoing data stream to the user + AXI4S.m m_axis_rdma_wr, + + // Incoming user commands from the HLS-stack + metaIntf.s s_rdma_wr_req, + + // Outgoing user commands to the user + metaIntf.m m_rdma_wr_req, + + // Incoming intrusion decision input + metaIntf.s s_intrusion_decision_in +); + +/////////////////////////////////////////////////////////////// +// +// Definition of registers required for managing accesses +// +/////////////////////////////////////////////////////////////// + +// Signal-array to cache two subsequent incoming DPI-decisions +logic dpi_decision[2]; + +// Signal to store which of the two DPI-decisions is currently active +logic dpi_currently_active; + +// Signal to store which of the two DPI-fields to load next +logic dpi_load_next; + +// Signal to show the currently active DPI-decision +logic current_dpi_acceptable; + +// Register stage with the prepared AXI-data signals that need to be forwarded +logic [511:0] axis_rdma_wr_data_inter; +logic [63:0] axis_rdma_wr_keep_inter; +logic axis_rdma_wr_valid_inter; +logic axis_rdma_wr_last_inter; + +// Register stage with the prepared control-signals that need to be forwarded +logic rdma_wr_req_valid_inter; +req_t rdma_wr_req_data_inter; + + +////////////////////////////////////////////////////////////////////// +// +// Combinatorial logic: Assign ready-signals +// +///////////////////////////////////////////////////////////////////// + +// ready signal will always be assigned - there's no blocking in this module +assign m_axis_rdma_wr.tready = s_axis_rdma_wr.tready; +assign m_rdma_wr_req.ready = s_rdma_wr_req.ready; + +// Forwarding the data interface +assign m_rdma_wr_req.valid = rdma_wr_req_valid_inter; +assign m_rdma_wr_req.data = rdma_wr_req_data_inter; + +// Assign the currently active dpi-acceptable signal +assign current_dpi_acceptable = dpi_currently_active ? dpi_decision[1].acceptable : dpi_decision[0].acceptable; + + +///////////////////////////////////////////////////////////////////// +// +// Sequential logic +// +//////////////////////////////////////////////////////////////////// + +always_ff @(posedge nclk) begin + if(!nresetn) begin + // RESET: Assign 0-values to all internal registers + dpi_decision[0] <= 0; + dpi_decision[1] <= 0; + dpi_currently_active <= 0; + dpi_load_next <= 0; + + // RESET: Assign 0-values to all intermediate registers + axis_rdma_wr_data_inter <= 512'b0; + axis_rdma_wr_keep_inter <= 64'b0; + axis_rdma_wr_valid_inter <= 1'b0; + axis_rdma_wr_last_inter <= 1'b0; + + rdma_wr_req_valid_inter <= 1'b0; + rdma_wr_req_data_inter <= 128'b0; + + end else begin + + // Wait for incoming DPI-decision + if(s_intrusion_decision_in.valid) begin + // Based on the load_next-signal, load the DPI-into the correct register + if(!dpi_load_next) begin + dpi_decision[0] <= s_intrusion_decision_in.data.acceptable; + end else begin + dpi_decision[1] <= s_intrusion_decision_in.data.acceptable; + end + + // Change the dpi_load_next so that the next DPI-decision can be stored in the other register + dpi_load_next <= !dpi_load_next; + end + + // Wait for incoming command + if(s_rdma_wr_req.valid) begin + // Check if the command is either a WRITE or a READ_RESPONSE. Everything else doesn't need treatment here + if(is_opcode_rd_resp(s_rdma_wr_req.data.opcode) || is_opcode_wr(s_rdma_wr_req.data.opcode)) begin + // Check the currenty active DPI-decision, then based on that decide whether to forward the original command or the one modified to raise an IRQ in the vFPGA + if(current_dpi_acceptable) begin + // If the current DPI-decision indicates that the packet is acceptable, just forward the command + rdma_wr_req_data_inter <= s_rdma_wr_req.data; + rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; + end else begin + // If the current DPI-decision indicates that the packet is not acceptable, forward a modified command with an opcode to raise an IRQ next + rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; + + rdma_wr_req_data_inter.opcode <= RC_ROCE_DPI_IRQ; // Change opcode so that it's raising an IRQ in the vFPGA + rdma_wr_req_data_inter.strm <= s_rdma_wr_req.data.strm; + rdma_wr_req_data_inter.mode <= s_rdma_wr_req.data.mode; + rdma_wr_req_data_inter.rdma <= s_rdma_wr_req.data.rdma; + rdma_wr_req_data_inter.remote <= s_rdma_wr_req.data.remote; + rdma_wr_req_data_inter.vfid <= s_rdma_wr_req.data.vfid; + rdma_wr_req_data_inter.pid <= s_rdma_wr_req.data.pid; + rdma_wr_req_data_inter.dest <= s_rdma_wr_req.data.dest; + rdma_wr_req_data_inter.last <= s_rdma_wr_req.data.last; + rdma_wr_req_data_inter.vaddr <= s_rdma_wr_req.data.vaddr; + rdma_wr_req_data_inter.len <= s_rdma_wr_req.data.len; + rdma_wr_req_data_inter.actv <= s_rdma_wr_req.data.actv; + rdma_wr_req_data_inter.host <= s_rdma_wr_req.data.host; + rdma_wr_req_data_inter.offs <= s_rdma_wr_req.data.offs; + rdma_wr_req_data_inter.rsrvd <= s_rdma_wr_req.data.rsrvd; + end + + end else begin + // If it's not a WRITE or READ_RESPONSE, just forward the commands on the corresponding interface + rdma_wr_req_data_inter <= s_rdma_wr_req.data; + rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; + end + end else begin + // If there's no active command, set the intermediate register to 0 + rdma_wr_req_data_inter <= 128'b0; + rdma_wr_req_valid_inter <= 1'b0; + end + end + + + // Wait for incoming data + if(s_axis_rdma_wr.tvalid) begin + // Check the currently active DPI-decision. + if(current_dpi_acceptable) begin + // If the current transmission is acceptable, just write the values in the intermediate register + axis_rdma_wr_data_inter <= s_axis_rdma_wr.tdata; + axis_rdma_wr_keep_inter <= s_axis_rdma_wr.tkeep; + axis_rdma_wr_last_inter <= s_axis_rdma_wr.tlast; + axis_rdma_wr_valid_inter <= s_axis_rdma_wr.tvalid; + end else begin + // If the current transmission is not acceptable, set the intermediate stage to 0 and thus drop the data + axis_rdma_wr_data_inter <= 512'b0; + axis_rdma_wr_keep_inter <= 64'b0; + axis_rdma_wr_last_inter <= 1'b0; + axis_rdma_wr_valid_inter <= 1'b0; + end + + // If a tlast is set, switch the pointer for the current DPI-decision + if(s_axis_rdma_wr.tlast) begin + dpi_currently_active <= !dpi_currently_active; + end + + end else begin + // If there's no active transmission right now, set the intermediate register stage to all 0 + axis_rdma_wr_data_inter <= 512'b0; + axis_rdma_wr_keep_inter <= 64'b0; + axis_rdma_wr_last_inter <= 1'b0; + axis_rdma_wr_valid_inter <= 1'b0; + end +end \ No newline at end of file diff --git a/hw/hdl/network/rdma/intrusion_detection_data_aggregator.sv b/hw/hdl/network/rdma/intrusion_detection_data_aggregator.sv index d9debf55..ced41414 100644 --- a/hw/hdl/network/rdma/intrusion_detection_data_aggregator.sv +++ b/hw/hdl/network/rdma/intrusion_detection_data_aggregator.sv @@ -30,6 +30,10 @@ module intrusion_detection_data_aggregator( localparam lp_opcode_write_middle = 8'h07; localparam lp_opcode_write_last = 8'h08; localparam lp_opcode_write_only = 8'h0a; + localparam lp_opcode_read_response_first = 8'h0d; + localparam lp_opcode_read_response_middle = 8'h0e; + localparam lp_opcode_read_response_last = 8'h0f; + localparam lp_opcode_read_response_only = 8'h10; /////////////////////////////////////////////////////////////////////////////////////// diff --git a/hw/hdl/network/rdma/intrusion_detection_decider.sv b/hw/hdl/network/rdma/intrusion_detection_decider.sv index 48c6dab2..62039a05 100644 --- a/hw/hdl/network/rdma/intrusion_detection_decider.sv +++ b/hw/hdl/network/rdma/intrusion_detection_decider.sv @@ -4,6 +4,9 @@ // ////////////////////////////////////////////////////////////////////////////////// +// Import the lynxTypes to be able to reference the datatypes +import lynxTypes::*; + module intrusion_detection_decider( // Incoming clock and reset input logic nclk, @@ -82,7 +85,7 @@ module intrusion_detection_decider( DataWord mlm_input_word; // Intermediate ML-decision aggregator - DecisionWord ml_decision_aggregator; + dpi_t ml_decision_aggregator; // Decision calculator logic decision_calculator; diff --git a/hw/hdl/network/rdma/payload_extractor.sv b/hw/hdl/network/rdma/payload_extractor.sv index 689d2c77..0af39ebc 100644 --- a/hw/hdl/network/rdma/payload_extractor.sv +++ b/hw/hdl/network/rdma/payload_extractor.sv @@ -33,6 +33,10 @@ module payload_extractor( localparam lp_opcode_write_middle = 8'h07; localparam lp_opcode_write_last = 8'h08; localparam lp_opcode_write_only = 8'h0a; + localparam lp_opcode_read_response_first = 8'h0d; + localparam lp_opcode_read_response_middle = 8'h0e; + localparam lp_opcode_read_response_last = 8'h0f; + localparam lp_opcode_read_response_only = 8'h10; //////////////////////////////////////////////////////////////////////// // @@ -136,7 +140,7 @@ module payload_extractor( assign qpn_extractor = m_axis_rx_data_i[287:264]; // Determine if the incoming packet is of WRITE-type and carries extractable payload - assign is_of_write_type = (opcode_extractor == lp_opcode_write_first) || (opcode_extractor == lp_opcode_write_last) || (opcode_extractor == lp_opcode_write_middle) || (opcode_extractor == lp_opcode_write_only); + assign is_of_write_type = (opcode_extractor == lp_opcode_write_first) || (opcode_extractor == lp_opcode_write_last) || (opcode_extractor == lp_opcode_write_middle) || (opcode_extractor == lp_opcode_write_only) || (opcode_extractor == lp_opcode_read_response_first) || (opcode_extractor == lp_opcode_read_response_middle) || (opcode_extractor == lp_opcode_read_response_last) || (opcode_extractor == lp_opcode_read_response_only); // Assure that the packet is actually RDMA assign marker_1 = (m_axis_rx_data_i[15:0] == 16'h0245); diff --git a/hw/hdl/network/rdma/roce_stack.sv b/hw/hdl/network/rdma/roce_stack.sv index 2f13f20c..aa092700 100644 --- a/hw/hdl/network/rdma/roce_stack.sv +++ b/hw/hdl/network/rdma/roce_stack.sv @@ -180,12 +180,14 @@ intrusion_detection_decider inst_intrusion_detection_decider ( dpi_transmission_dropper inst_dpi_transmission_dropper ( .nclk(nclk), .nresetn(nresetn), + // Data interface: Data to be written from HLS-stack to RDMA-output .s_axis_rdma_wr(m_axis_rdma_wr_roce_2_dpi_dropper), .m_axis_rdma_wr(m_axis_rdma_wr), + // Command interface: Write-command from HLS to RDMA-output .s_rdma_wr_req(m_rdma_wr_req_roce_2_dpi_dropper), .m_rdma_wr_req(rdma_wr_req), - .s_rdma_ack(m_rdma_ack_roce_2_dpi_dropper), - .m_rdma_ack(m_rdma_ack) + // Intrusion decision input + .s_intrusion_decision_in(intrusion_decision_stack2dropper) ); diff --git a/scripts/wr_hdl/template_gen/lynx_pkg_tmplt.txt b/scripts/wr_hdl/template_gen/lynx_pkg_tmplt.txt index e291e6ae..a1518d84 100644 --- a/scripts/wr_hdl/template_gen/lynx_pkg_tmplt.txt +++ b/scripts/wr_hdl/template_gen/lynx_pkg_tmplt.txt @@ -172,6 +172,7 @@ package lynxTypes; parameter integer IRQ_PFAULT = 3; parameter integer IRQ_NOTIFY = 4; parameter integer IRQ_RCNFG = 5; + parameter integer IRQ_DPI = 6; // Data parameter integer ADDR_BITS = 64; @@ -262,6 +263,9 @@ package lynxTypes; parameter integer RC_RDMA_READ_RESP_ONLY = 5'h10; parameter integer RC_ACK = 5'h11; + // Add an additional opcode specifically for notifications coming from the RoCE-stack up to the vFPGA to raise IRQs if required + parameter integer RC_ROCE_DPI_IRQ = 5'h12; + parameter integer RDMA_ACK_BITS = 64; parameter integer RDMA_ACK_QPN_BITS = 10; parameter integer RDMA_ACK_SYNDROME_BITS = 8; @@ -424,6 +428,12 @@ package lynxTypes; req_t req_2; // wr, remote } dreq_t; + // New datatype for DPI-decisions + typedef struct packed{ + logic[23:0] QPN; + logic acceptable; + } dpi_t; + typedef struct packed { logic [OPCODE_BITS-1:0] opcode; logic [STRM_BITS-1:0] strm; diff --git a/sw/include/bThread.hpp b/sw/include/bThread.hpp index e5660a6d..1fd6e02f 100644 --- a/sw/include/bThread.hpp +++ b/sw/include/bThread.hpp @@ -195,6 +195,7 @@ class bThread { */ bool doArpLookup(uint32_t ip_addr); bool writeQpContext(uint32_t port); + void readQpContext(uint32_t port); void connSync(bool client); void connClose(bool client); diff --git a/sw/include/cLib.hpp b/sw/include/cLib.hpp index d2fb7f1a..3b6a5e3b 100644 --- a/sw/include/cLib.hpp +++ b/sw/include/cLib.hpp @@ -198,6 +198,9 @@ class cLib { // Write context and connection to the configuration registers cthread->writeQpContext(port); + // Read back the QP-context to check the content of the registers + cthread->readQpContext(port); + // ARP lookup to get the MAC-address for the remote QP IP-address # ifdef VERBOSE std::cout << "cLib: Initiate an Arp-lookup for the IP-address " << cthread->getQpair()->remote.ip_addr << std::endl; @@ -320,6 +323,9 @@ class cLib { // Write context and connection to the config-space of Coyote cthread->writeQpContext(port); + + // Read back the context to check the registers + cthread->readQpContext(port); // Perform an ARP lookup cthread->doArpLookup(cthread->getQpair()->remote.ip_addr); diff --git a/sw/src/bThread.cpp b/sw/src/bThread.cpp index fef04ad0..a0d73ab6 100644 --- a/sw/src/bThread.cpp +++ b/sw/src/bThread.cpp @@ -1107,6 +1107,34 @@ bool bThread::writeQpContext(uint32_t port) { return false; } +/** + * @brief Read back the queue pair context from the register space to see if everything worked as expected + */ +void bThread::readQpContext(uint32_t port) { + // Get an array to write values into + uint64_t offs[6]; + // Check that both RDMA and AVX are activated since that is the only readback-method that I'm going to support here + if(fcnfg.en_rdma) { + if(fcnfg.en_avx) { + offs[0] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_1)], 0x0); + offs[1] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_1)], 0x1); + offs[2] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_1)], 0x2); + + offs[3] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_2)], 0x0); + offs[4] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_2)], 0x1); + offs[5] = _mm256_extract_epi64(cnfg_reg_avx[static_cast(CnfgAvxRegs::RDMA_CTX_REG_2)], 0x2); + + std::cout << "bThread: Called readQpContext on a RDMA-enabled design." << std::endl; + std::cout << " - bThread - offs[0] " << offs[0] << std::endl; + std::cout << " - bThread - offs[1] " << offs[1] << std::endl; + std::cout << " - bThread - offs[2] " << offs[2] << std::endl; + std::cout << " - bThread - offs[3] " << offs[3] << std::endl; + std::cout << " - bThread - offs[4] " << offs[4] << std::endl; + std::cout << " - bThread - offs[5] " << offs[5] << std::endl; + } + } +} + /** * @brief Set connection */