Merge pull request #759 from pbalcer/adapters-update

[ADAPTERS] update to latest from sycl

.github/docker/README.md

@@ -0,0 +1,42 @@
+# Content
+
+Dockerfiles and scripts placed in this directory are intended to be used as
+development process vehicles and part of continuous integration process.
+
+Images built out of those recipes may be used with Docker or podman as
+development environment.
+
+# How to build docker image
+
+To build docker image on local machine execute:
+
+```sh
+docker build -t ur:ubuntu-22.04 -f ./ubuntu-22.04.Dockerfile .
+```
+
+To set any build time variable (e.g., an optional ARG from docker recipe), add to the command (after `build`), e.g.:
+
+```sh
+ --build-arg SKIP_DPCPP_BUILD=1
+```
+
+One other example of using these extra build arguments are proxy settings. They are required for accessing network
+(e.g., to download dependencies within docker), if a host is using a proxy server. Example usage:
+
+```sh
+ --build-arg https_proxy=http://proxy.com:port --build-arg http_proxy=http://proxy.com:port 
+```
+
+# How to use docker image
+
+To run docker container (using the previously built image) execute:
+
+```sh
+docker run --shm-size=4G -v /your/workspace/path/:/opt/workspace:z -w /opt/workspace/ -it ur:ubuntu-22.04 /bin/bash
+```
+
+To set (or override) any docker environment variable, add to the command (after `run`):
+
+```sh
+ -e ENV_VARIABLE=VALUE
+```

.github/docker/install_dpcpp.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright (C) 2023 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#
+# install_dpcpp.sh - unpacks DPC++ in ${DPCPP_PATH}, to be used while building UR
+#
+
+set -e
+
+if [ "${SKIP_DPCPP_BUILD}" ]; then
+ echo "Variable 'SKIP_DPCPP_BUILD' is set; skipping building DPC++"
+ exit
+fi
+
+apt-get install -y --no-install-recommends \
+ libncurses5
+
+mkdir -p ${DPCPP_PATH}
+wget -O ${DPCPP_PATH}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/sycl-nightly%2F20230626/dpcpp-compiler.tar.gz
+tar -xvf ${DPCPP_PATH}/dpcpp_compiler.tar.gz -C ${DPCPP_PATH}/

.github/docker/install_libbacktrace.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright (C) 2023 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#
+# install_libbacktrace.sh - builds and installs tracing library
+#
+
+set -e
+
+if [ "${SKIP_LIBBACKTRACE_BUILD}" ]; then
+ echo "Variable 'SKIP_LIBBACKTRACE_BUILD' is set; skipping building libbacktrace"
+ exit
+fi
+
+git clone https://github.com/ianlancetaylor/libbacktrace.git
+pushd libbacktrace
+./configure
+make -j$(nproc)
+make install -j$(nproc)
+popd

.github/docker/ubuntu-22.04.Dockerfile

@@ -0,0 +1,70 @@
+# Copyright (C) 2023 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#
+# Dockerfile - image with all Unified Runtime dependencies.
+#
+
+# Pull base image
+FROM registry.hub.docker.com/library/ubuntu:22.04
+
+# Set environment variables
+ENV OS ubuntu
+ENV OS_VER 22.04
+ENV NOTTY 1
+ENV DEBIAN_FRONTEND noninteractive
+
+# Additional parameters to build docker without building components.
+# These ARGs can be set in docker building phase and are used
+# within bash scripts (executed within docker).
+ARG SKIP_DPCPP_BUILD
+ARG SKIP_LIBBACKTRACE_BUILD
+
+# Base development packages
+ARG BASE_DEPS="\
+ build-essential \
+ cmake \
+ git"
+
+# Unified Runtime's dependencies
+ARG UR_DEPS="\
+ doxygen \
+ python3 \
+ python3-pip"
+
+# Unified Runtime's dependencies (installed via pip)
+ARG UR_PYTHON_DEPS="\
+ clang-format==15.0.7"
+
+# Miscellaneous for our builds/CI (optional)
+ARG MISC_DEPS="\
+ clang \
+ sudo \
+ wget \
+ whois"
+
+# Update and install required packages
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ ${BASE_DEPS} \
+ ${UR_DEPS} \
+ ${MISC_DEPS} \
+ && apt-get clean all
+
+RUN pip3 install ${UR_PYTHON_DEPS}
+
+# Install DPC++
+COPY install_dpcpp.sh install_dpcpp.sh
+ENV DPCPP_PATH=/opt/dpcpp
+RUN ./install_dpcpp.sh
+
+# Install libbacktrace
+COPY install_libbacktrace.sh install_libbacktrace.sh
+RUN ./install_libbacktrace.sh
+
+# Add a new (non-root) 'user'
+ENV USER user
+ENV USERPASS pass
+RUN useradd -m $USER -g sudo -p `mkpasswd $USERPASS`

.github/workflows/cmake.yml

@@ -11,6 +11,7 @@ jobs:
  build_type: [Debug, Release]
  compiler: [{c: gcc, cxx: g++}]
  libbacktrace: ['-DVAL_USE_LIBBACKTRACE_BACKTRACE=OFF']
+ pool_tracking: ['-DUMF_ENABLE_POOL_TRACKING=ON', '-DUMF_ENABLE_POOL_TRACKING=OFF']
  include:
  - os: 'ubuntu-22.04'
  build_type: Release
@@ -53,9 +54,15 @@ jobs:
  cd libbacktrace
  ./configure
  make
- sudo make install 
+ sudo make install
  cd ..
 
+ - name: Download DPC++
+ run: |
+ sudo apt install libncurses5
+ wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/sycl-nightly%2F20230626/dpcpp-compiler.tar.gz
+ tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz
+
  - name: Configure CMake
  run: >
  cmake
@@ -66,7 +73,9 @@ jobs:
  -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
  -DUR_BUILD_TESTS=ON
  -DUR_FORMAT_CPP_STYLE=ON
+ -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
  ${{matrix.libbacktrace}}
+ ${{matrix.pool_tracking}}
 
  - name: Generate source from spec, check for uncommitted diff
  if: matrix.os == 'ubuntu-22.04'
@@ -77,7 +86,7 @@ jobs:
 
  - name: Test
  working-directory: ${{github.workspace}}/build
- run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|uma|loader|validation|tracing|unit|urtrace"
+ run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|umf|loader|validation|tracing|unit|urtrace"
 
  adapter-build:
  name: Build - Adapters on Ubuntu
@@ -165,4 +174,35 @@ jobs:
 
  - name: Test
  working-directory: ${{github.workspace}}/build
- run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|uma|loader|validation|tracing|unit|urtrace"
+ run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|umf|loader|validation|tracing|unit|urtrace"
+
+
+ macos-build:
+ name: Build - MacOS
+ strategy:
+ matrix:
+ os: ['macos-12', 'macos-13']
+ runs-on: ${{matrix.os}}
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: 3.9
+
+ - name: Install prerequisites
+ run: python3 -m pip install -r third_party/requirements.txt
+
+ - name: Configure CMake
+ run: >
+ cmake
+ -B${{github.workspace}}/build
+ -DUR_ENABLE_TRACING=ON
+ -DUR_DEVELOPER_MODE=ON
+ -DCMAKE_BUILD_TYPE=Release
+ -DUR_BUILD_TESTS=ON
+ -DUR_FORMAT_CPP_STYLE=ON
+ -DUMF_ENABLE_POOL_TRACKING=ON
+ - name: Build
+ run: cmake --build ${{github.workspace}}/build -j $(nproc)

.github/workflows/codeql.yml

@@ -27,7 +27,7 @@ jobs:
  run: pip install -r third_party/requirements.txt
 
  - name: Configure CMake
- run: cmake -B ${{github.workspace}}/build -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_ENABLE_TRACING=ON -DUR_BUILD_TOOLS=ON
+ run: cmake -B ${{github.workspace}}/build -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_ENABLE_TRACING=ON -DUR_BUILD_TOOLS=ON -DUMF_ENABLE_POOL_TRACKING=ON
 
  - name: Build
  run: cmake --build ${{github.workspace}}/build -j $(nproc)
@@ -61,7 +61,7 @@ jobs:
  run: python3 -m pip install -r third_party/requirements.txt
 
  - name: Configure CMake
- run: cmake -B ${{github.workspace}}/build -DCMAKE_POLICY_DEFAULT_CMP0094=NEW -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_ENABLE_TRACING=ON -DUR_BUILD_TOOLS=ON
+ run: cmake -B ${{github.workspace}}/build -DCMAKE_POLICY_DEFAULT_CMP0094=NEW -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_ENABLE_TRACING=ON -DUR_BUILD_TOOLS=ON -DUMF_ENABLE_POOL_TRACKING=ON
 
  - name: Build
  run: cmake --build ${{github.workspace}}/build -j $(nproc) --config Release

.github/workflows/coverity.yml

@@ -26,7 +26,7 @@ jobs:
  linux:
  name: Coverity
  runs-on: ubuntu-latest
- 
+
  steps:
  - name: Clone the git repo
  uses: actions/checkout@v3
@@ -35,12 +35,12 @@ jobs:
  run: |
  sudo apt-get update
  sudo apt-get install -y doxygen
- 
+
  - name: Install pip packages
  run: pip install -r third_party/requirements.txt
 
  - name: Configure CMake
- run: cmake -B $WORKDIR/build -DUR_ENABLE_TRACING=ON -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_FORMAT_CPP_STYLE=ON
+ run: cmake -B $WORKDIR/build -DUR_ENABLE_TRACING=ON -DUR_DEVELOPER_MODE=ON -DUR_BUILD_TESTS=ON -DUR_FORMAT_CPP_STYLE=ON -DUMF_ENABLE_POOL_TRACKING=ON
 
  - name: Generate source from spec, check for uncommitted diff
  run: |

CMakeLists.txt

@@ -14,6 +14,10 @@ include(CTest)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 include(helpers)
 
+if(CMAKE_SYSTEM_NAME STREQUAL Darwin)
+ set(CMAKE_FIND_FRAMEWORK NEVER)
+endif()
+
 find_package(Python3 COMPONENTS Interpreter)
 
 set(CMAKE_CXX_STANDARD 17)
@@ -26,10 +30,12 @@ option(UR_FORMAT_CPP_STYLE "format code style of C++ sources" OFF)
 option(UR_USE_ASAN "enable AddressSanitizer" OFF)
 option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF)
 option(UR_USE_MSAN "enable MemorySanitizer" OFF)
-option(UMA_BUILD_SHARED_LIBRARY "Build UMA as shared library" OFF)
+option(UR_USE_TSAN "enable ThreadSanitizer" OFF)
+option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF)
 option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF)
 option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF)
 option(UR_BUILD_TOOLS "build ur tools" ON)
+option(UMF_ENABLE_POOL_TRACKING "Build UMF with pool tracking" OFF)
 option(UR_BUILD_ADAPTER_L0 "build level 0 adapter from SYCL" OFF)
 option(UR_BUILD_ADAPTER_CUDA "build cuda adapter from SYCL" OFF)
 option(UR_BUILD_ADAPTER_HIP "build hip adapter from SYCL" OFF)
@@ -51,14 +57,7 @@ if(MSVC)
  set(CUSTOM_COMMAND_BINARY_DIR ${CUSTOM_COMMAND_BINARY_DIR}/$<CONFIG>)
 endif()
 
-# CXX flags setup
 if(NOT MSVC)
- add_compile_options(-fPIC -Wall -Wpedantic
- $<$<CXX_COMPILER_ID:GNU>:-fdiagnostics-color=always>
- $<$<CXX_COMPILER_ID:Clang>:-fcolor-diagnostics>)
- if(UR_DEVELOPER_MODE)
- add_compile_options(-Werror -fno-omit-frame-pointer)
- endif()
  # Determine if libstdc++ is being used.
  check_cxx_source_compiles("
  #include <array>
@@ -74,21 +73,13 @@ if(NOT MSVC)
  # is available we still need to link this library.
  link_libraries(stdc++fs)
  endif()
-elseif(MSVC)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MP")
- add_compile_options(/MP /W3 /MD$<$<CONFIG:Debug>:d>)
-
- if(UR_DEVELOPER_MODE)
- add_compile_options(/WX)
- endif()
 endif()
 
 if(UR_ENABLE_TRACING)
  add_compile_definitions(UR_ENABLE_TRACING)
 
  # fetch xpti proxy library for the tracing layer
- FetchContentSparse_Declare(xpti https://github.com/intel/llvm.git "sycl-nightly/20230304" "xpti")
+ FetchContentSparse_Declare(xpti https://github.com/intel/llvm.git "sycl-nightly/20230703" "xpti")
  FetchContent_MakeAvailable(xpti)
 
  # set -fPIC for xpti since we are linking it with a shared library
@@ -100,7 +91,7 @@ if(UR_ENABLE_TRACING)
  set(XPTI_DIR ${xpti_SOURCE_DIR})
  set(XPTI_ENABLE_TESTS OFF CACHE INTERNAL "Turn off xptifw tests")
 
- FetchContentSparse_Declare(xptifw https://github.com/intel/llvm.git "sycl-nightly/20230304" "xptifw")
+ FetchContentSparse_Declare(xptifw https://github.com/intel/llvm.git "sycl-nightly/20230703" "xptifw")
 
  FetchContent_MakeAvailable(xptifw)
 
@@ -130,6 +121,10 @@ if(UR_USE_UBSAN)
  add_sanitizer_flag(undefined)
 endif()
 
+if(UR_USE_TSAN)
+ add_sanitizer_flag(thread)
+endif()
+
 if(UR_USE_MSAN)
  message(WARNING "MemorySanitizer requires that all code is built with
  its instrumentation, otherwise false positives are possible.