From 0af0d86c33800346f9d187576fb36fcb258e313f Mon Sep 17 00:00:00 2001 From: HailoRT-Automation <98901220+HailoRT-Automation@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:14:22 +0200 Subject: [PATCH] v4.16.0 (#10) Co-authored-by: HailoRT-Automation --- common/hailo_pcie_version.h | 2 +- common/vdma_common.c | 82 +++++++--- common/vdma_common.h | 2 + download_firmware.cmd | 2 +- download_firmware.sh | 2 +- linux-port/linux/bitops.h | 14 -- linux-port/linux/bug.h | 21 --- linux-port/linux/circ_buf.h | 12 -- linux-port/linux/delay.h | 29 ---- linux-port/linux/errno.h | 53 ------- linux-port/linux/io.h | 41 ----- linux-port/linux/kernel.h | 19 --- linux-port/linux/ktime.h | 11 -- linux-port/linux/scatterlist.h | 44 ------ linux-port/linux/timekeeping.h | 30 ---- linux-port/linux/types.h | 94 ----------- linux/integrated_nnc/Kbuild | 2 + .../include/hailo_integrated_nnc_version.h | 2 +- linux/integrated_nnc/src/dram_vdma.c | 3 +- linux/integrated_nnc/src/fw_control.c | 2 +- linux/integrated_nnc/src/main.c | 4 +- linux/pcie/src/fops.c | 12 +- linux/pcie/src/fops.h | 6 + linux/pcie/src/pcie.c | 37 ++--- linux/pcie/src/utils.c | 1 + linux/utils/compact.h | 21 ++- linux/vdma/ioctl.c | 7 + linux/vdma/memory.c | 147 ++++++++++++++---- linux/vdma/memory.h | 2 +- linux/vdma/vdma.c | 16 +- linux/vdma/vdma.h | 11 +- 31 files changed, 265 insertions(+), 466 deletions(-) delete mode 100644 linux-port/linux/bitops.h delete mode 100644 linux-port/linux/bug.h delete mode 100644 linux-port/linux/circ_buf.h delete mode 100644 linux-port/linux/delay.h delete mode 100644 linux-port/linux/errno.h delete mode 100644 linux-port/linux/io.h delete mode 100644 linux-port/linux/kernel.h delete mode 100644 linux-port/linux/ktime.h delete mode 100644 linux-port/linux/scatterlist.h delete mode 100644 linux-port/linux/timekeeping.h delete mode 100644 linux-port/linux/types.h diff --git a/common/hailo_pcie_version.h b/common/hailo_pcie_version.h index 0e1492c..63af361 100644 --- a/common/hailo_pcie_version.h +++ b/common/hailo_pcie_version.h @@ -7,7 +7,7 @@ #define _HAILO_COMMON_PCIE_VERSION_H_ #define HAILO_DRV_VER_MAJOR 4 -#define HAILO_DRV_VER_MINOR 15 +#define HAILO_DRV_VER_MINOR 16 #define HAILO_DRV_VER_REVISION 0 #endif /* _HAILO_COMMON_PCIE_VERSION_H_ */ \ No newline at end of file diff --git a/common/vdma_common.c b/common/vdma_common.c index 64b4623..477e240 100644 --- a/common/vdma_common.c +++ b/common/vdma_common.c @@ -11,6 +11,7 @@ #include #include #include +#include #define CHANNEL_BASE_OFFSET(channel_index, direction) (((direction) == HAILO_DMA_TO_DEVICE) ? \ @@ -73,23 +74,61 @@ static uint8_t get_channel_id(uint8_t channel_index) } } +static int program_descriptors_in_chunk( + dma_addr_t chunk_addr, + unsigned int chunk_size, + struct hailo_vdma_descriptors_list *desc_list, + uint32_t *desc_index, + uint16_t desc_page_size, + uint32_t max_desc_index, + uint8_t data_id, + uint8_t channel_id, + encode_desc_dma_address_t address_encoder) +{ + const uint32_t desc_per_chunk = DIV_ROUND_UP(chunk_size, desc_page_size); + struct hailo_vdma_descriptor *dma_desc = NULL; + uint32_t index = 0; + uint64_t encoded_addr = 0; + + for (index = 0; index < desc_per_chunk; index++) { + if (*desc_index > max_desc_index) { + return -ERANGE; + } + + encoded_addr = address_encoder(chunk_addr, channel_id); + if (INVALID_VDMA_ADDRESS == encoded_addr) { + return -EFAULT; + } + + dma_desc = &desc_list->desc_list[*desc_index % desc_list->desc_count]; + hailo_vdma_program_descriptor(dma_desc, encoded_addr, desc_page_size, data_id); + + chunk_addr += desc_page_size; + (*desc_index)++; + } + + return 0; +} + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4127) +#endif int hailo_vdma_program_descriptors_list( struct hailo_desc_list_bind_vdma_buffer_params *params, struct hailo_vdma_descriptors_list *desc_list, struct sg_table *buffer, + dma_addr_t mmio_dma_address, + uint32_t size, encode_desc_dma_address_t address_encoder, uint8_t data_id) { - struct hailo_vdma_descriptor *dma_desc = NULL; const uint8_t channel_id = get_channel_id(params->channel_index); uint32_t desc_index = 0; uint32_t max_desc_index = 0; - uint32_t desc_in_sg = 0; - dma_addr_t desc_buffer_addr = 0; - uint64_t encoded_addr = 0; - uint32_t desc_per_sg = 0; struct scatterlist *sg_entry = NULL; unsigned int i = 0; + int ret = 0; if (!is_powerof2(params->desc_page_size)) { return -EFAULT; @@ -105,30 +144,27 @@ int hailo_vdma_program_descriptors_list( params->starting_desc + desc_list->desc_count - 1 : desc_list->desc_count - 1; desc_index = params->starting_desc; - for_each_sgtable_dma_sg(buffer, sg_entry, i) { - desc_per_sg = DIV_ROUND_UP(sg_dma_len(sg_entry), params->desc_page_size); - desc_buffer_addr = sg_dma_address(sg_entry); - for (desc_in_sg = 0; desc_in_sg < desc_per_sg; desc_in_sg++) { - if (desc_index > max_desc_index) { - return -ERANGE; - } - - encoded_addr = address_encoder(desc_buffer_addr, channel_id); - if (INVALID_VDMA_ADDRESS == encoded_addr) { - return -EFAULT; + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) && (INVALID_VDMA_ADDRESS != mmio_dma_address)) { + ret = program_descriptors_in_chunk(mmio_dma_address, size, desc_list, + &desc_index, params->desc_page_size, max_desc_index, data_id, channel_id, address_encoder); + if (ret < 0) { + return ret; + } + } else { + for_each_sgtable_dma_sg(buffer, sg_entry, i) { + ret = program_descriptors_in_chunk(sg_dma_address(sg_entry), sg_dma_len(sg_entry), desc_list, + &desc_index, params->desc_page_size, max_desc_index, data_id, channel_id, address_encoder); + if (ret < 0) { + return ret; } - - dma_desc = &desc_list->desc_list[desc_index % desc_list->desc_count]; - hailo_vdma_program_descriptor(dma_desc, encoded_addr, - params->desc_page_size, data_id); - - desc_buffer_addr += params->desc_page_size; - desc_index++; } } return 0; } +#if defined(_MSC_VER) +#pragma warning(pop) +#endif static void hailo_vdma_push_timestamp(struct hailo_channel_interrupt_timestamp_list *timestamp_list, struct hailo_resource *vdma_registers, size_t channel_index, enum hailo_dma_data_direction direction) diff --git a/common/vdma_common.h b/common/vdma_common.h index e49cd3b..c4f8e9a 100644 --- a/common/vdma_common.h +++ b/common/vdma_common.h @@ -69,6 +69,8 @@ int hailo_vdma_program_descriptors_list( struct hailo_desc_list_bind_vdma_buffer_params *params, struct hailo_vdma_descriptors_list *desc_list, struct sg_table *buffer, + dma_addr_t mmio_dma_address, + uint32_t size, encode_desc_dma_address_t address_encoder, uint8_t data_id); diff --git a/download_firmware.cmd b/download_firmware.cmd index cb3240e..50882b7 100644 --- a/download_firmware.cmd +++ b/download_firmware.cmd @@ -2,7 +2,7 @@ @ECHO OFF set BASE_URI=https://hailo-hailort.s3.eu-west-2.amazonaws.com -set HRT_VERSION=4.15.0 +set HRT_VERSION=4.16.0 set FW_DIR=Hailo8/%HRT_VERSION%/FW set FW=hailo8_fw.%HRT_VERSION%.bin diff --git a/download_firmware.sh b/download_firmware.sh index 2535cd5..2d95c85 100755 --- a/download_firmware.sh +++ b/download_firmware.sh @@ -2,7 +2,7 @@ set -e readonly BASE_URI="https://hailo-hailort.s3.eu-west-2.amazonaws.com" -readonly HRT_VERSION=4.15.0 +readonly HRT_VERSION=4.16.0 readonly FW_AWS_DIR="Hailo8/${HRT_VERSION}/FW" readonly FW="hailo8_fw.${HRT_VERSION}.bin" diff --git a/linux-port/linux/bitops.h b/linux-port/linux/bitops.h deleted file mode 100644 index 69a6f2d..0000000 --- a/linux-port/linux/bitops.h +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_BITOPS_H_ -#define _HAILO_LINUX_BITOPS_H_ - -#define BITS_PER_LONG 32 -#define UL(x) ((unsigned long)x) -#define BIT_MASK(nr) (UL(1) << ((nr) % BITS_PER_LONG)) -#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) - -#endif /* _HAILO_LINUX_BITOPS_H_ */ \ No newline at end of file diff --git a/linux-port/linux/bug.h b/linux-port/linux/bug.h deleted file mode 100644 index 518f455..0000000 --- a/linux-port/linux/bug.h +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_PORT_BUG_H_ -#define _HAILO_LINUX_PORT_BUG_H_ - -#ifdef _MSC_VER - -#define BUG_ON(condition) if (condition) { KeBugCheckEx(0xE4A12000, (ULONG_PTR)__FILE__, __LINE__, 0, 0); } else {} - -#elif defined(__QNX__) - -#define BUG_ON(condition) if (condition) { printf("CRITICAL ERROR CAUGHT\n"); exit(0); } else {} - -#else -#error "Unsupported Platform" -#endif - -#endif /* _HAILO_LINUX_PORT_BUG_H_ */ \ No newline at end of file diff --git a/linux-port/linux/circ_buf.h b/linux-port/linux/circ_buf.h deleted file mode 100644 index acd46f7..0000000 --- a/linux-port/linux/circ_buf.h +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_CIRC_BUF_H_ -#define _HAILO_LINUX_CIRC_BUF_H_ - -#define CIRC_CNT(head,tail,size) (((head) - (tail)) & ((size)-1)) -#define CIRC_SPACE(head,tail,size) CIRC_CNT((tail),((head)+1),(size)) - -#endif /* _HAILO_LINUX_CIRC_BUF_H_ */ \ No newline at end of file diff --git a/linux-port/linux/delay.h b/linux-port/linux/delay.h deleted file mode 100644 index 269198d..0000000 --- a/linux-port/linux/delay.h +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_DELAY_H_ -#define _HAILO_LINUX_DELAY_H_ - -#ifdef _MSC_VER - -void FORCEINLINE msleep(ULONG milliseconds) -{ - LARGE_INTEGER li; - li.QuadPart = -10000LL * milliseconds; - KeDelayExecutionThread(KernelMode, false, &li); -} - -#elif defined(__QNX__) - -inline static void msleep(int milliseconds) -{ - usleep(milliseconds * 1000); -} - -#else -#error "Unsupported Platform" -#endif - -#endif /* _HAILO_LINUX_DELAY_H_ */ \ No newline at end of file diff --git a/linux-port/linux/errno.h b/linux-port/linux/errno.h deleted file mode 100644 index d64acd4..0000000 --- a/linux-port/linux/errno.h +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_PORT_ERRNO_H_ -#define _HAILO_LINUX_PORT_ERRNO_H_ - -#ifdef _MSC_VER - -#if !defined(EIO) -#define EIO 5 -#endif - -#if !defined(ENOMEM) -#define ENOMEM 12 -#endif - -#if !defined(EFAULT) -#define EFAULT 14 -#endif - -#if !defined(ENODEV) -#define ENODEV 19 -#endif - -#if !defined(EINVAL) -#define EINVAL 22 -#endif - -#if !defined(ERANGE) -#define ERANGE 34 -#endif - - -#if !defined(ETIMEDOUT) -#define ETIMEDOUT 60 -#endif - -#if !defined(ENOSYS) -#define ENOSYS 88 -#endif - - -#elif defined(__QNX__) - -#include - -#else -#error "Unsupported Platform" -#endif - -#endif /* _HAILO_LINUX_PORT_ERRNO_H_ */ \ No newline at end of file diff --git a/linux-port/linux/io.h b/linux-port/linux/io.h deleted file mode 100644 index 3d49f5e..0000000 --- a/linux-port/linux/io.h +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_PORT_IO_H_ -#define _HAILO_LINUX_PORT_IO_H_ - -#include - -static uint8_t ioread8(volatile void* src) -{ - return *(volatile uint8_t*)src; -} - -static uint16_t ioread16(volatile void* src) -{ - return *(volatile uint16_t*)src; -} - -static uint32_t ioread32(volatile void* src) -{ - return *(volatile uint32_t*)src; -} - -static void iowrite8(uint8_t value, volatile void* dest) -{ - *(volatile uint8_t*)dest = value; -} - -static void iowrite16(uint16_t value, volatile void* dest) -{ - *(volatile uint16_t*)dest = value; -} - -static void iowrite32(uint32_t value, volatile void* dest) -{ - *(volatile uint32_t *)dest = value; -} - -#endif /* _HAILO_LINUX_PORT_IO_H_ */ \ No newline at end of file diff --git a/linux-port/linux/kernel.h b/linux-port/linux/kernel.h deleted file mode 100644 index 8c9991a..0000000 --- a/linux-port/linux/kernel.h +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_KERNEL_H_ -#define _HAILO_LINUX_KERNEL_H_ - -#include - - -#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0])) - -#define IS_ALIGNED(p, n) (((uintptr_t)(p) & (n - 1)) == 0) -#define ALIGN(x,a) __ALIGN_MASK(x,(typeof(x))(a)-1) -#define __ALIGN_MASK(x,mask) (((x)+(mask))&~(mask)) - - -#endif /* _HAILO_LINUX_KERNEL_H_ */ \ No newline at end of file diff --git a/linux-port/linux/ktime.h b/linux-port/linux/ktime.h deleted file mode 100644 index 905e4a5..0000000 --- a/linux-port/linux/ktime.h +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_KTIMEG_H_ -#define _HAILO_LINUX_KTIMEG_H_ - -// Empty header for include - -#endif /* _HAILO_LINUX_KTIME_H_ */ \ No newline at end of file diff --git a/linux-port/linux/scatterlist.h b/linux-port/linux/scatterlist.h deleted file mode 100644 index 87172f0..0000000 --- a/linux-port/linux/scatterlist.h +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_SCATTERLIST_H_ -#define _HAILO_LINUX_SCATTERLIST_H_ - -#include - -struct scatterlist { - unsigned int length; - dma_addr_t dma_address; -}; - -struct sg_table { - struct scatterlist *sgl; /* the list */ - unsigned int nents; /* number of mapped entries */ -}; - -/* - * Loop over each sg element, following the pointer to a new list if necessary - */ -#define for_each_sg(sglist, sg, nr, __i) \ - for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg)) - -/* - * Loop over each sg element in the given *DMA mapped* sg_table object. - * Please use sg_dma_address(sg) and sg_dma_len(sg) to extract DMA addresses - * of the each element. - */ -#define for_each_sgtable_dma_sg(sgt, sg, i) \ - for_each_sg((sgt)->sgl, sg, (sgt)->nents, i) - -static inline struct scatterlist *sg_next(struct scatterlist *sg) -{ - sg++; - return sg; -} - -#define sg_dma_address(sg) ((sg)->dma_address) -#define sg_dma_len(sg) ((sg)->length) - -#endif /* _HAILO_LINUX_SCATTERLIST_H_ */ \ No newline at end of file diff --git a/linux-port/linux/timekeeping.h b/linux-port/linux/timekeeping.h deleted file mode 100644 index 6699857..0000000 --- a/linux-port/linux/timekeeping.h +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2023 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_TIMEKEEPING_H_ -#define _HAILO_LINUX_TIMEKEEPING_H_ - -#ifdef _MSC_VER - -static uint64_t FORCEINLINE ktime_get_ns() -{ - ULONG64 qpcTimeStamp; - ULONG64 val = KeQueryInterruptTimePrecise(&qpcTimeStamp); - val *= 100LL; - return val; -} - -#elif defined(__QNX__) - -inline static uint64_t ktime_get_ns() -{ - return (ClockCycles() / SYSPAGE_ENTRY(qtime)->cycles_per_sec) * 1000000; -} - -#else -#error "Unsupported Platform" -#endif - -#endif /* _HAILO_LINUX_TIMEKEEPING_H_ */ \ No newline at end of file diff --git a/linux-port/linux/types.h b/linux-port/linux/types.h deleted file mode 100644 index 4a9e005..0000000 --- a/linux-port/linux/types.h +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * Copyright (c) 2019-2022 Hailo Technologies Ltd. All rights reserved. - **/ - -#ifndef _HAILO_LINUX_TYPES_H_ -#define _HAILO_LINUX_TYPES_H_ - -#ifdef _MSC_VER - -#include - -typedef ULONG uint32_t; -typedef UCHAR uint8_t; -typedef USHORT uint16_t; -typedef ULONGLONG uint64_t; -typedef uint64_t u64; -typedef uint32_t u32; -typedef uint16_t u16; -typedef uint8_t u8; - -typedef uint64_t dma_addr_t; - -#if !defined(U32_MAX) -#define U32_MAX ((u32)~0U) -#endif - -#ifndef UINT_MAX -#define UINT_MAX (~0U) -#endif - -#define _CSTDINT_ -#define _VCRUNTIME_H - -#include -#include - - -#define false FALSE -#define true TRUE - -// On msvc, we don't have likely and unlikely - so we just don't this optimization -#define unlikely(x) (x) -#define likely(x) (x) - -#elif defined(__QNX__) - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// TODO: HRT-6138 - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -// At the moment dont do anything special for QNX -typedef uint64_t u64; -typedef uint32_t u32; -typedef uint16_t u16; -typedef uint8_t u8; - -// Define dma_addr_t as physical memory address for QNX -typedef uint64_t dma_addr_t; - -#if !defined(INT_MAX) -#define INT_MAX 0x7FFFFFFF -#endif - -#ifndef UINT_MAX -#define UINT_MAX (~0U) -#endif - -#if !defined(U32_MAX) -#define U32_MAX ((u32)~0U) -#endif - -#else - -#error "Unsupported Platform" - -#endif - -#include "hailo_ioctl_common.h" - -#endif /* _HAILO_LINUX_TYPES_H_ */ diff --git a/linux/integrated_nnc/Kbuild b/linux/integrated_nnc/Kbuild index 0af0acb..26133df 100644 --- a/linux/integrated_nnc/Kbuild +++ b/linux/integrated_nnc/Kbuild @@ -36,5 +36,7 @@ ccflags-y += -Werror ccflags-y += -I$(src)/include ccflags-y += -I$(src)/$(COMMON_SRC_DIRECTORY) ccflags-y += -I$(src)/$(BASE_INCLUDE_DIRECTORY) +# TODO: Support HAILO_SUPPORT_MMIO_DMA_MAPPING in windows and linux pcie drivers (HRT-12516, HRT-12517) +ccflags-y += -DHAILO_SUPPORT_MMIO_DMA_MAPPING clean-files := $(hailo_integrated_nnc-objs) diff --git a/linux/integrated_nnc/include/hailo_integrated_nnc_version.h b/linux/integrated_nnc/include/hailo_integrated_nnc_version.h index a11e8c5..7c7b3bb 100755 --- a/linux/integrated_nnc/include/hailo_integrated_nnc_version.h +++ b/linux/integrated_nnc/include/hailo_integrated_nnc_version.h @@ -9,7 +9,7 @@ #include #define HAILO_DRV_VER_MAJOR 4 -#define HAILO_DRV_VER_MINOR 15 +#define HAILO_DRV_VER_MINOR 16 #define HAILO_DRV_VER_REVISION 0 #define HAILO_DRV_VER __stringify(HAILO_DRV_VER_MAJOR) "." __stringify(HAILO_DRV_VER_MINOR) "." __stringify(HAILO_DRV_VER_REVISION) diff --git a/linux/integrated_nnc/src/dram_vdma.c b/linux/integrated_nnc/src/dram_vdma.c index dfa0339..5f2d63f 100644 --- a/linux/integrated_nnc/src/dram_vdma.c +++ b/linux/integrated_nnc/src/dram_vdma.c @@ -4,8 +4,9 @@ **/ #include "board.h" -#include "vdma/vdma.h" +#include "dram_vdma.h" #include "integrated_nnc_utils.h" +#include "vdma/vdma.h" #include "utils/logs.h" #include diff --git a/linux/integrated_nnc/src/fw_control.c b/linux/integrated_nnc/src/fw_control.c index 2a81331..5d59502 100644 --- a/linux/integrated_nnc/src/fw_control.c +++ b/linux/integrated_nnc/src/fw_control.c @@ -94,7 +94,7 @@ long hailo_fw_control(struct hailo_board *board, unsigned long arg, bool *should return err; } -void rx_callback(struct mbox_client *cl, void *mssg) +static void rx_callback(struct mbox_client *cl, void *mssg) { struct fw_control *control = container_of(cl, struct fw_control, mbox_client); complete(&control->response_ready); diff --git a/linux/integrated_nnc/src/main.c b/linux/integrated_nnc/src/main.c index fec7057..f1ab130 100644 --- a/linux/integrated_nnc/src/main.c +++ b/linux/integrated_nnc/src/main.c @@ -22,7 +22,7 @@ #include "fw_logger.h" #include "dram_vdma.h" #include "utils/logs.h" - +#include "utils/compact.h" #define DRIVER_NAME "hailo_integrated_nnc" #define DEVICE_NODE_NAME "hailo_integrated_nnc" @@ -134,7 +134,7 @@ static int driver_probe(struct platform_device *pdev) } /* Creating struct class */ - class = class_create(THIS_MODULE, "hailo_chardev"); + class = class_create_compat("hailo_chardev"); if (IS_ERR(class)) { err = PTR_ERR(class); hailo_err(board, "Failed creating class. err: %d", err); diff --git a/linux/pcie/src/fops.c b/linux/pcie/src/fops.c index 6863d22..f0eae39 100644 --- a/linux/pcie/src/fops.c +++ b/linux/pcie/src/fops.c @@ -325,7 +325,7 @@ static long hailo_read_log_ioctl(struct hailo_pcie_board *pBoard, unsigned long return 0; } -void firmware_notification_irq_handler(struct hailo_pcie_board *board) +static void firmware_notification_irq_handler(struct hailo_pcie_board *board) { struct hailo_notification_wait *notif_wait_cursor = NULL; int err = 0; @@ -433,7 +433,7 @@ static long hailo_add_notification_wait(struct hailo_pcie_board *board, struct f return 0; } -long hailo_read_notification_ioctl(struct hailo_pcie_board *pBoard, unsigned long arg, struct file *filp, +static long hailo_read_notification_ioctl(struct hailo_pcie_board *pBoard, unsigned long arg, struct file *filp, bool* should_up_board_mutex) { long err = 0; @@ -486,7 +486,7 @@ long hailo_read_notification_ioctl(struct hailo_pcie_board *pBoard, unsigned lon return err; } -long hailo_disable_notification(struct hailo_pcie_board *pBoard, struct file *filp) +static long hailo_disable_notification(struct hailo_pcie_board *pBoard, struct file *filp) { struct hailo_notification_wait *cursor = NULL; @@ -504,7 +504,7 @@ long hailo_disable_notification(struct hailo_pcie_board *pBoard, struct file *fi return 0; } -int hailo_fw_control(struct hailo_pcie_board *pBoard, unsigned long arg, bool* should_up_board_mutex) +static int hailo_fw_control(struct hailo_pcie_board *pBoard, unsigned long arg, bool* should_up_board_mutex) { struct hailo_fw_control *command = &pBoard->fw_control.command; long completion_result = 0; @@ -562,7 +562,7 @@ int hailo_fw_control(struct hailo_pcie_board *pBoard, unsigned long arg, bool* s return err; } -long hailo_query_device_properties(struct hailo_pcie_board *board, unsigned long arg) +static long hailo_query_device_properties(struct hailo_pcie_board *board, unsigned long arg) { struct hailo_device_properties props = { .desc_max_page_size = board->desc_max_page_size, @@ -582,7 +582,7 @@ long hailo_query_device_properties(struct hailo_pcie_board *board, unsigned long return 0; } -long hailo_query_driver_info(struct hailo_pcie_board *board, unsigned long arg) +static long hailo_query_driver_info(struct hailo_pcie_board *board, unsigned long arg) { struct hailo_driver_info info = { .major_version = HAILO_DRV_VER_MAJOR, diff --git a/linux/pcie/src/fops.h b/linux/pcie/src/fops.h index 6f926ef..d0afc93 100644 --- a/linux/pcie/src/fops.h +++ b/linux/pcie/src/fops.h @@ -12,4 +12,10 @@ long hailo_pcie_fops_unlockedioctl(struct file* filp, unsigned int cmd, unsigned int hailo_pcie_fops_mmap(struct file* filp, struct vm_area_struct *vma); int hailo_pcie_driver_down(struct hailo_pcie_board *board); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +irqreturn_t hailo_irqhandler(int irq, void* dev_id, struct pt_regs *regs); +#else +irqreturn_t hailo_irqhandler(int irq, void* dev_id); +#endif + #endif /* _HAILO_PCI_FOPS_H_ */ diff --git a/linux/pcie/src/pcie.c b/linux/pcie/src/pcie.c index c14e0b1..28e85d1 100644 --- a/linux/pcie/src/pcie.c +++ b/linux/pcie/src/pcie.c @@ -59,12 +59,6 @@ static struct semaphore g_hailo_add_board_mutex = __SEMAPHORE_INITIALIZER(g_hail #define HAILO_IRQ_FLAGS (IRQF_SHARED | IRQF_DISABLED) #else #define HAILO_IRQ_FLAGS (IRQF_SHARED) -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) -irqreturn_t hailo_irqhandler(int irq, void* dev_id, struct pt_regs *regs); -#else -irqreturn_t hailo_irqhandler(int irq, void* dev_id); #endif /* **************************** @@ -250,7 +244,7 @@ static int hailo_pcie_disable_aspm(struct hailo_pcie_board *board, u16 state, bo return 0; } -void hailo_pcie_insert_board(struct hailo_pcie_board* pBoard) +static void hailo_pcie_insert_board(struct hailo_pcie_board* pBoard) { uint32_t index = 0; struct hailo_pcie_board *pCurrent, *pNext; @@ -284,7 +278,7 @@ void hailo_pcie_insert_board(struct hailo_pcie_board* pBoard) return; } -void hailo_pcie_remove_board(struct hailo_pcie_board* pBoard) +static void hailo_pcie_remove_board(struct hailo_pcie_board* pBoard) { down(&g_hailo_add_board_mutex); if (pBoard) @@ -617,7 +611,7 @@ static bool is_kmalloc_dma_capable(struct device *dev) return capable; } -int hailo_get_allocation_mode(struct pci_dev *pdev, enum hailo_allocation_mode *allocation_mode) +static int hailo_get_allocation_mode(struct pci_dev *pdev, enum hailo_allocation_mode *allocation_mode) { // Check if module paramater was given to override driver choice if (HAILO_NO_FORCE_BUFFER != force_allocation_from_driver) { @@ -695,12 +689,6 @@ static int hailo_pcie_probe(struct pci_dev* pDev, const struct pci_device_id* id goto probe_release_pcie_resources; } - err = hailo_get_allocation_mode(pDev, &pBoard->allocation_mode); - if (err < 0) { - pci_err(pDev, "Failed determining allocation of buffers from driver. error type: %d\n", err); - goto probe_release_pcie_resources; - } - pBoard->interrupts_enabled = false; init_completion(&pBoard->fw_loaded_completion); @@ -726,6 +714,13 @@ static int hailo_pcie_probe(struct pci_dev* pDev, const struct pci_device_id* id goto probe_release_pcie_resources; } + // Checks the dma mask => it must be called after the device's dma_mask is set by hailo_pcie_vdma_controller_init + err = hailo_get_allocation_mode(pDev, &pBoard->allocation_mode); + if (err < 0) { + pci_err(pDev, "Failed determining allocation of buffers from driver. error type: %d\n", err); + goto probe_release_pcie_resources; + } + err = hailo_activate_board(pBoard); if (err < 0) { hailo_err(pBoard, "Failed activating board %d\n", err); @@ -769,7 +764,7 @@ static int hailo_pcie_probe(struct pci_dev* pDev, const struct pci_device_id* id return err; } -void hailo_pcie_remove(struct pci_dev* pDev) +static void hailo_pcie_remove(struct pci_dev* pDev) { struct hailo_pcie_board* pBoard = (struct hailo_pcie_board*) pci_get_drvdata(pDev); struct hailo_notification_wait *cursor = NULL; @@ -957,24 +952,24 @@ static struct pci_driver hailo_pci_driver = MODULE_DEVICE_TABLE (pci, hailo_pcie_id_table); -int hailo_pcie_register_chrdev(unsigned int major, const char *name) +static int hailo_pcie_register_chrdev(unsigned int major, const char *name) { int char_major; char_major = register_chrdev(major, name, &hailo_pcie_fops); - chardev_class = class_create(THIS_MODULE, "hailo_chardev"); + chardev_class = class_create_compat("hailo_chardev"); return char_major; } -void hailo_pcie_unregister_chrdev(unsigned int major, const char *name) +static void hailo_pcie_unregister_chrdev(unsigned int major, const char *name) { class_destroy(chardev_class); unregister_chrdev(major, name); } -int __init hailo_pcie_module_init(void) +static int __init hailo_pcie_module_init(void) { int err; @@ -998,7 +993,7 @@ int __init hailo_pcie_module_init(void) return 0; } -void __exit hailo_pcie_module_exit(void) +static void __exit hailo_pcie_module_exit(void) { pr_notice(DRIVER_NAME ": Exit module.\n"); diff --git a/linux/pcie/src/utils.c b/linux/pcie/src/utils.c index bef1aff..c4fab4a 100644 --- a/linux/pcie/src/utils.c +++ b/linux/pcie/src/utils.c @@ -10,6 +10,7 @@ #include "hailo_pcie_version.h" #include "pcie.h" +#include "utils.h" #include "utils/logs.h" diff --git a/linux/utils/compact.h b/linux/utils/compact.h index 8439710..c33967d 100644 --- a/linux/utils/compact.h +++ b/linux/utils/compact.h @@ -10,6 +10,12 @@ #include #include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0) +#define class_create_compat class_create +#else +#define class_create_compat(name) class_create(THIS_MODULE, name) +#endif + #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) #define pci_printk(level, pdev, fmt, arg...) \ dev_printk(level, &(pdev)->dev, fmt, ##arg) @@ -23,21 +29,22 @@ #define pci_dbg(pdev, fmt, arg...) dev_dbg(&(pdev)->dev, fmt, ##arg) #endif - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) #define get_user_pages_compact get_user_pages +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +#define get_user_pages_compact(start, nr_pages, gup_flags, pages) \ + get_user_pages(start, nr_pages, gup_flags, pages, NULL) #elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) -#define get_user_pages_compact(start, nr_pages, gup_flags, pages, vmas) \ - get_user_pages(current, current->mm, start, nr_pages, gup_flags, pages, vmas) +#define get_user_pages_compact(start, nr_pages, gup_flags, pages) \ + get_user_pages(current, current->mm, start, nr_pages, gup_flags, pages, NULL) #else static inline long get_user_pages_compact(unsigned long start, unsigned long nr_pages, - unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas) + unsigned int gup_flags, struct page **pages) { int write = !!((gup_flags & FOLL_WRITE) == FOLL_WRITE); int force = !!((gup_flags & FOLL_FORCE) == FOLL_FORCE); return get_user_pages(current, current->mm, start, nr_pages, write, force, - pages, vmas); + pages, NULL); } #endif diff --git a/linux/vdma/ioctl.c b/linux/vdma/ioctl.c index f30e621..12b0e67 100644 --- a/linux/vdma/ioctl.c +++ b/linux/vdma/ioctl.c @@ -360,6 +360,11 @@ long hailo_vdma_buffer_sync(struct hailo_vdma_file_context *context, struct hail return -EINVAL; } + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) && (INVALID_VDMA_ADDRESS != mapped_buffer->mmio_dma_address)) { + // MMIO buffers don't need to be sync'd + return 0; + } + if ((sync_info.sync_type != HAILO_SYNC_FOR_CPU) && (sync_info.sync_type != HAILO_SYNC_FOR_DEVICE)) { hailo_dev_err(controller->dev, "Invalid sync_type given for vdma buffer sync.\n"); return -EINVAL; @@ -487,6 +492,8 @@ long hailo_desc_list_bind_vdma_buffer(struct hailo_vdma_file_context *context, s return hailo_vdma_program_descriptors_list(&configure_info, &descriptors_buffer->desc_list, &mapped_buffer->sg_table, + mapped_buffer->mmio_dma_address, + mapped_buffer->size, controller->ops->encode_desc_dma_address, data_id ); diff --git a/linux/vdma/memory.c b/linux/vdma/memory.c index a53bbbb..ba3e125 100644 --- a/linux/vdma/memory.c +++ b/linux/vdma/memory.c @@ -14,18 +14,26 @@ #define SGL_MAX_SEGMENT_SIZE (0x10000) +// See linux/mm.h +#define MMIO_AND_NO_PAGES_VMA_MASK (VM_IO | VM_PFNMAP) +static int map_mmio_address(void __user* user_address, uint32_t size, struct vm_area_struct *vma, + struct device *dev, enum dma_data_direction direction, dma_addr_t *mmio_dma_address); static int prepare_sg_table(struct sg_table *sg_table, void __user* user_address, uint32_t size, - struct hailo_vdma_low_memory_buffer *allocated_vdma_buffer); + struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer); static void clear_sg_table(struct sg_table *sgt, bool put_pages); struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, void __user *user_address, size_t size, enum dma_data_direction direction, - struct hailo_vdma_low_memory_buffer *allocated_vdma_buffer) + struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer) { int ret = -EINVAL; struct hailo_vdma_buffer *mapped_buffer = NULL; + // Either sgt will be a zero initialized struct and mmio_dma_address != INVALID_VDMA_ADDRESS + // or sgt will be mapped to dma and mmio_dma_address will be INVALID_VDMA_ADDRESS + dma_addr_t mmio_dma_address = INVALID_VDMA_ADDRESS; struct sg_table sgt = {0}; + struct vm_area_struct *vma = NULL; mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL); if (NULL == mapped_buffer) { @@ -34,17 +42,43 @@ struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, goto cleanup; } - ret = prepare_sg_table(&sgt, user_address, size, allocated_vdma_buffer); - if (ret < 0) { - dev_err(dev, "failed to set sg list for user buffer %d\n", ret); - goto free_buffer_struct; + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING)) { + vma = find_vma(current->mm, (uintptr_t)user_address); + if (NULL == vma) { + dev_err(dev, "no vma for virt_addr/size = 0x%08lx/0x%08zx\n", (uintptr_t)user_address, size); + ret = -EFAULT; + goto cleanup; + } } - sgt.nents = dma_map_sg(dev, sgt.sgl, sgt.orig_nents, direction); - if (0 == sgt.nents) { - dev_err(dev, "failed to map sg list for user buffer\n"); - ret = -ENXIO; - goto clear_sg_table; + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) && + (MMIO_AND_NO_PAGES_VMA_MASK == (vma->vm_flags & MMIO_AND_NO_PAGES_VMA_MASK))) { + // user_address represents memory mapped I/O and isn't backed by 'struct page' (only by pure pfn) + if (NULL != low_mem_driver_allocated_buffer) { + // low_mem_driver_allocated_buffer are backed by regular 'struct page' addresses, just in low memory + dev_err(dev, "low_mem_driver_allocated_buffer shouldn't be provided with an mmio address\n"); + ret = -EINVAL; + goto free_buffer_struct; + } + + ret = map_mmio_address(user_address, size, vma, dev, direction, &mmio_dma_address); + if (ret < 0) { + dev_err(dev, "failed to map mmio address %d\n", ret); + goto free_buffer_struct; + } + } else { + // user_address is a standard 'struct page' backed memory address + ret = prepare_sg_table(&sgt, user_address, size, low_mem_driver_allocated_buffer); + if (ret < 0) { + dev_err(dev, "failed to set sg list for user buffer %d\n", ret); + goto free_buffer_struct; + } + sgt.nents = dma_map_sg(dev, sgt.sgl, sgt.orig_nents, direction); + if (0 == sgt.nents) { + dev_err(dev, "failed to map sg list for user buffer\n"); + ret = -ENXIO; + goto clear_sg_table; + } } kref_init(&mapped_buffer->kref); @@ -53,12 +87,16 @@ struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, mapped_buffer->size = size; mapped_buffer->data_direction = direction; mapped_buffer->sg_table = sgt; - mapped_buffer->driver_buffer_handle = allocated_vdma_buffer ? - (allocated_vdma_buffer->handle) : INVALID_DRIVER_HANDLE_VALUE; + mapped_buffer->low_mem_driver_allocated_buffer_handle = low_mem_driver_allocated_buffer ? + low_mem_driver_allocated_buffer->handle : + INVALID_DRIVER_HANDLE_VALUE; + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING)) { + mapped_buffer->mmio_dma_address = mmio_dma_address; + } return mapped_buffer; clear_sg_table: - clear_sg_table(&sgt, (NULL == allocated_vdma_buffer)); + clear_sg_table(&sgt, (NULL == low_mem_driver_allocated_buffer)); free_buffer_struct: kfree(mapped_buffer); cleanup: @@ -67,14 +105,15 @@ struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, static void unmap_buffer(struct kref *kref) { - bool put_pages = false; struct hailo_vdma_buffer *buf = container_of(kref, struct hailo_vdma_buffer, kref); - dma_unmap_sg(buf->device, buf->sg_table.sgl, buf->sg_table.orig_nents, - buf->data_direction); - - put_pages = ((INVALID_DRIVER_HANDLE_VALUE) == buf->driver_buffer_handle); - clear_sg_table(&buf->sg_table, put_pages); + if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) && (INVALID_VDMA_ADDRESS != buf->mmio_dma_address)) { + // TODO: need dma_unmap_resource here? add it if we add dma_map_resource (HRT-12521) + } else { + const bool put_pages = ((INVALID_DRIVER_HANDLE_VALUE) == buf->low_mem_driver_allocated_buffer_handle); + dma_unmap_sg(buf->device, buf->sg_table.sgl, buf->sg_table.orig_nents, buf->data_direction); + clear_sg_table(&buf->sg_table, put_pages); + } } void hailo_vdma_buffer_get(struct hailo_vdma_buffer *buf) @@ -176,7 +215,7 @@ int hailo_vdma_low_memory_buffer_alloc(size_t size, struct hailo_vdma_low_memory pages = kcalloc(pages_count, sizeof(*pages), GFP_KERNEL); if (NULL == pages) { - pr_err("Failed to allocate pages for buffer (size %zu)", size); + pr_err("Failed to allocate pages for buffer (size %zu)\n", size); ret = -ENOMEM; goto cleanup; } @@ -298,8 +337,58 @@ void hailo_vdma_clear_continuous_buffer_list(struct hailo_vdma_file_context *con } } +// Assumes the provided user_address belongs to the vma and that MMIO_AND_NO_PAGES_VMA_MASK bits are set under +// vma->vm_flags. This is validated in hailo_vdma_buffer_map, and won't be checked here +static int map_mmio_address(void __user* user_address, uint32_t size, struct vm_area_struct *vma, + struct device *dev, enum dma_data_direction direction, dma_addr_t *mmio_dma_address) +{ + int ret = -EINVAL; + unsigned long i = 0; + unsigned long pfn = 0; + unsigned long next_pfn = 0; + phys_addr_t phys_addr = 0; + const uintptr_t virt_addr = (uintptr_t)user_address; + const uint32_t vma_size = vma->vm_end - vma->vm_start + 1; + const uintptr_t num_pages = PFN_UP(virt_addr + size) - PFN_DOWN(virt_addr); + + // Check that the vma that was marked as MMIO_AND_NO_PAGES_VMA_MASK is big enough + if (vma_size < size) { + pr_err("vma (%u bytes) smaller than provided buffer (%u bytes)\n", vma_size, size); + return -EINVAL; + } + + // Get the physical address of user_address + ret = follow_pfn(vma, virt_addr, &pfn); + if (ret) { + pr_err("follow_pfn failed with %d\n", ret); + return ret; + } + phys_addr = __pfn_to_phys(pfn) + offset_in_page(virt_addr); + + // Make sure the physical memory is contiguous + for (i = 1; i < num_pages; ++i) { + ret = follow_pfn(vma, virt_addr + (i << PAGE_SHIFT), &next_pfn); + if (ret < 0) { + pr_err("follow_pfn failed with %d\n", ret); + return ret; + } + if (next_pfn != pfn + 1) { + pr_err("non-contiguous physical memory\n"); + return -EFAULT; + } + pfn = next_pfn; + } + + // Finally - phys_addr to dma + // TODO: need dma_map_resource here? doesn't work currently (we get dma_mapping_error on the returned dma addr) + // (HRT-12521) + *mmio_dma_address = (dma_addr_t)phys_addr; + + return 0; +} + static int prepare_sg_table(struct sg_table *sg_table, void __user *user_address, uint32_t size, - struct hailo_vdma_low_memory_buffer *allocated_vdma_buffer) + struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer) { int ret = -EINVAL; int pinned_pages = 0; @@ -314,10 +403,11 @@ static int prepare_sg_table(struct sg_table *sg_table, void __user *user_address return -ENOMEM; } - // Check whether mapping user allocated buffer or driver allocated buffer - if (NULL == allocated_vdma_buffer) { + // Check whether mapping user allocated buffer or driver allocated low memory buffer + if (NULL == low_mem_driver_allocated_buffer) { mmap_read_lock(current->mm); - pinned_pages = get_user_pages_compact((unsigned long)user_address, npages, FOLL_WRITE | FOLL_FORCE, pages, NULL); + pinned_pages = get_user_pages_compact((unsigned long)user_address, + npages, FOLL_WRITE | FOLL_FORCE, pages); mmap_read_unlock(current->mm); if (pinned_pages < 0) { @@ -329,17 +419,16 @@ static int prepare_sg_table(struct sg_table *sg_table, void __user *user_address ret = -EINVAL; goto release_pages; } - } else { // Check to make sure in case user provides wrong buffer - if (npages != allocated_vdma_buffer->pages_count) { + if (npages != low_mem_driver_allocated_buffer->pages_count) { pr_err("Received wrong amount of pages %zu to map expected %zu\n", - npages, allocated_vdma_buffer->pages_count); + npages, low_mem_driver_allocated_buffer->pages_count); ret = -EINVAL; goto exit; } for (i = 0; i < npages; i++) { - pages[i] = virt_to_page(allocated_vdma_buffer->pages_address[i]); + pages[i] = virt_to_page(low_mem_driver_allocated_buffer->pages_address[i]); } } diff --git a/linux/vdma/memory.h b/linux/vdma/memory.h index 0babb64..8f185a5 100644 --- a/linux/vdma/memory.h +++ b/linux/vdma/memory.h @@ -13,7 +13,7 @@ struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, void __user *user_address, size_t size, enum dma_data_direction direction, - struct hailo_vdma_low_memory_buffer *allocated_vdma_buffer); + struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer); void hailo_vdma_buffer_get(struct hailo_vdma_buffer *buf); void hailo_vdma_buffer_put(struct hailo_vdma_buffer *buf); diff --git a/linux/vdma/vdma.c b/linux/vdma/vdma.c index da0f4b8..8ba1940 100644 --- a/linux/vdma/vdma.c +++ b/linux/vdma/vdma.c @@ -11,6 +11,14 @@ #include "utils/logs.h" #include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#include +#else +#include +#endif + static struct hailo_vdma_engine* init_vdma_engines(struct device *dev, struct hailo_resource *channel_registers_per_engine, size_t engines_count) @@ -78,6 +86,10 @@ int hailo_vdma_controller_init(struct hailo_vdma_controller *controller, return err; } + if (get_dma_ops(controller->dev)) { + hailo_dev_notice(controller->dev, "Probing: Using specialized dma_ops=%ps", get_dma_ops(controller->dev)); + } + return 0; } @@ -224,7 +236,9 @@ static int desc_list_mmap(struct hailo_vdma_controller *controller, return err; } } else { - unsigned long pfn = virt_to_phys(vdma_descriptors_buffer->kernel_address) >> PAGE_SHIFT; + const unsigned long pfn = is_vmalloc_addr(vdma_descriptors_buffer->kernel_address) ? + vmalloc_to_pfn(vdma_descriptors_buffer->kernel_address) : + virt_to_phys(vdma_descriptors_buffer->kernel_address) >> PAGE_SHIFT; err = remap_pfn_range(vma, vma->vm_start, pfn, vsize, vma->vm_page_prot); if (err != 0) { hailo_dev_err(controller->dev, " vdma_mmap failed remap_pfn_range %d\n", err); diff --git a/linux/vdma/vdma.h b/linux/vdma/vdma.h index d8fc21d..dd94217 100644 --- a/linux/vdma/vdma.h +++ b/linux/vdma/vdma.h @@ -40,10 +40,17 @@ struct hailo_vdma_buffer { enum dma_data_direction data_direction; struct sg_table sg_table; // Can be INVALID_DRIVER_HANDLE_VALUE if the buffer is allocated by the user - uintptr_t driver_buffer_handle; + uintptr_t low_mem_driver_allocated_buffer_handle; + // This address will be used if user_address represents memory mapped I/O and isn't backed by 'struct page' (only + // by pure pfn), as designated by a vma's flags being set to VM_IO and VM_PFNMAP respectively (see linux/mm.h). + // In this case, mmio_dma_address will be used instead of the sg_table above, since dma_sg_* functions require + // standard memory addresses backed by 'struct page'. + // For standard memory addresses (backed by 'struct page'), mmio_dma_address will be INVALID_VDMA_ADDRESS and + // sg_table will be used. + dma_addr_t mmio_dma_address; }; -// Continous buffer that holds a descriptor list. +// Continuous buffer that holds a descriptor list. struct hailo_descriptors_list_buffer { struct list_head descriptors_buffer_list; uintptr_t handle;