Skip to content

Commit

Permalink
Merge branch 'main' into yinggeh-DLIS-7113-support-histogram-metric-type
Browse files Browse the repository at this point in the history
  • Loading branch information
@yinggeh
yinggeh authored Aug 16, 2024
2 parents ef797b3 + 7a158a4 commit 66912aa
Show file tree
Hide file tree
Showing 4 changed files with 228 additions and 37 deletions.
16 changes: 6 additions & 10 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,17 +125,13 @@ FetchContent_Declare(

# Some libs are installed to ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib64 instead
# of ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/{LIB}/lib on Centos
set (LIB_DIR "lib")
# /etc/os-release does not exist on Windows
if(EXISTS "/etc/os-release")
file(STRINGS /etc/os-release DISTRO REGEX "^NAME=")
string(REGEX REPLACE "NAME=\"(.*)\"" "\\1" DISTRO "${DISTRO}")
message(STATUS "Distro Name: ${DISTRO}")
if(DISTRO MATCHES "CentOS.*")
set(LIB_DIR "lib")
if(LINUX)
file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE")
if(${DISTRO_ID_LIKE} MATCHES "rhel|centos")
set (LIB_DIR "lib64")
endif()
endif()

endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos")
endif(LINUX)
set(TRITON_CORE_HEADERS_ONLY OFF)

FetchContent_MakeAvailable(repo-third-party repo-core)
Expand Down
216 changes: 196 additions & 20 deletions build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import sys
from inspect import getsourcefile

import distro
import requests

#
Expand Down Expand Up @@ -117,7 +118,17 @@ def fail_if(p, msg):
def target_platform():
if FLAGS.target_platform is not None:
return FLAGS.target_platform
return platform.system().lower()
platform_string = platform.system().lower()
if platform_string == "linux":
# Need to inspect the /etc/os-release file to get
# the distribution of linux
id_like_list = distro.like().split()
if "debian" in id_like_list:
return "linux"
else:
return "rhel"
else:
return platform_string


def target_machine():
Expand Down Expand Up @@ -649,7 +660,8 @@ def onnxruntime_cmake_args(images, library_paths):
]

# TRITON_ENABLE_GPU is already set for all backends in backend_cmake_args()
if FLAGS.enable_gpu:
# TODO: TPRD-334 TensorRT extension is not currently supported by our manylinux build
if FLAGS.enable_gpu and target_platform() != "rhel":
cargs.append(
cmake_backend_enable(
"onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_TENSORRT", True
Expand Down Expand Up @@ -680,8 +692,11 @@ def onnxruntime_cmake_args(images, library_paths):
)
)

if (target_machine() != "aarch64") and (
TRITON_VERSION_MAP[FLAGS.version][3] is not None
# TODO: TPRD-333 OpenVino extension is not currently supported by our manylinux build
if (
(target_machine() != "aarch64")
and (target_platform() != "rhel")
and (TRITON_VERSION_MAP[FLAGS.version][3] is not None)
):
cargs.append(
cmake_backend_enable(
Expand All @@ -697,7 +712,7 @@ def onnxruntime_cmake_args(images, library_paths):
)
)

if target_platform() == "igpu":
if (target_platform() == "igpu") or (target_platform() == "rhel"):
cargs.append(
cmake_backend_arg(
"onnxruntime",
Expand Down Expand Up @@ -833,8 +848,31 @@ def install_dcgm_libraries(dcgm_version, target_machine):
)
return ""
else:
if target_machine == "aarch64":
return """
# RHEL has the same install instructions for both aarch64 and x86
if target_platform() == "rhel":
if target_machine == "aarch64":
return """
ENV DCGM_VERSION {}
# Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo \\
&& dnf clean expire-cache \\
&& dnf install -y datacenter-gpu-manager-{}
""".format(
dcgm_version, dcgm_version
)
else:
return """
ENV DCGM_VERSION {}
# Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo \\
&& dnf clean expire-cache \\
&& dnf install -y datacenter-gpu-manager-{}
""".format(
dcgm_version, dcgm_version
)
else:
if target_machine == "aarch64":
return """
ENV DCGM_VERSION {}
# Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
RUN curl -o /tmp/cuda-keyring.deb \\
Expand All @@ -844,10 +882,10 @@ def install_dcgm_libraries(dcgm_version, target_machine):
&& apt-get update \\
&& apt-get install -y datacenter-gpu-manager=1:{}
""".format(
dcgm_version, dcgm_version
)
else:
return """
dcgm_version, dcgm_version
)
else:
return """
ENV DCGM_VERSION {}
# Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
RUN curl -o /tmp/cuda-keyring.deb \\
Expand All @@ -857,8 +895,106 @@ def install_dcgm_libraries(dcgm_version, target_machine):
&& apt-get update \\
&& apt-get install -y datacenter-gpu-manager=1:{}
""".format(
dcgm_version, dcgm_version
)
dcgm_version, dcgm_version
)


def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap):
df = """
ARG TRITON_VERSION={}
ARG TRITON_CONTAINER_VERSION={}
ARG BASE_IMAGE={}
""".format(
argmap["TRITON_VERSION"],
argmap["TRITON_CONTAINER_VERSION"],
argmap["BASE_IMAGE"],
)

df += """
FROM ${BASE_IMAGE}
ARG TRITON_VERSION
ARG TRITON_CONTAINER_VERSION
"""
df += """
# Install docker docker buildx
RUN yum install -y ca-certificates curl gnupg yum-utils \\
&& yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo \\
&& yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# && yum install -y docker.io docker-buildx-plugin
# libcurl4-openSSL-dev is needed for GCS
# python3-dev is needed by Torchvision
# python3-pip and libarchive-dev is needed by python backend
# libxml2-dev is needed for Azure Storage
# scons is needed for armnn_tflite backend build dep
RUN yum install -y \\
ca-certificates \\
autoconf \\
automake \\
git \\
gperf \\
re2-devel \\
openssl-devel \\
libtool \\
libcurl-devel \\
libb64-devel \\
gperftools-devel \\
patchelf \\
python3.11-devel \\
python3-pip \\
python3-setuptools \\
rapidjson-devel \\
python3-scons \\
pkg-config \\
unzip \\
wget \\
zlib-devel \\
libarchive-devel \\
libxml2-devel \\
numactl-devel \\
wget
RUN pip3 install --upgrade pip \\
&& pip3 install --upgrade \\
wheel \\
setuptools \\
docker \\
virtualenv
# Install boost version >= 1.78 for boost::span
# Current libboost-dev apt packages are < 1.78, so install from tar.gz
RUN wget -O /tmp/boost.tar.gz \\
https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz \\
&& (cd /tmp && tar xzf boost.tar.gz) \\
&& mv /tmp/boost_1_80_0/boost /usr/include/boost
# Server build requires recent version of CMake (FetchContent required)
# Might not need this if the installed version of cmake is high enough for our build.
# RUN apt update -q=2 \\
# && apt install -y gpg wget \\
# && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\
# && . /etc/os-release \\
# && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\
# && apt-get update -q=2 \\
# && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7*
"""
if FLAGS.enable_gpu:
df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine())
df += """
ENV TRITON_SERVER_VERSION ${TRITON_VERSION}
ENV NVIDIA_TRITON_SERVER_VERSION ${TRITON_CONTAINER_VERSION}
"""

df += """
WORKDIR /workspace
RUN rm -fr *
COPY . .
ENTRYPOINT []
"""

with open(os.path.join(ddir, dockerfile_name), "w") as dfile:
dfile.write(df)


def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
Expand Down Expand Up @@ -1161,7 +1297,29 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
fi \\
&& [ `id -u $TRITON_SERVER_USER` -eq 1000 ] \\
&& [ `id -g $TRITON_SERVER_USER` -eq 1000 ]
""".format(
gpu_enabled=gpu_enabled
)

# This
if target_platform() == "rhel":
df += """
# Common dpeendencies.
RUN yum install -y \\
git \\
gperf \\
re2-devel \\
openssl-devel \\
libtool \\
libcurl-devel \\
libb64-devel \\
gperftools-devel \\
patchelf \\
wget \\
numactl-devel
"""
else:
df += """
# Ensure apt-get won't prompt for selecting options
ENV DEBIAN_FRONTEND=noninteractive
Expand All @@ -1184,12 +1342,14 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
wget \\
{backend_dependencies} \\
&& rm -rf /var/lib/apt/lists/*
""".format(
backend_dependencies=backend_dependencies
)

df += """
# Set TCMALLOC_RELEASE_RATE for users setting LD_PRELOAD with tcmalloc
ENV TCMALLOC_RELEASE_RATE 200
""".format(
gpu_enabled=gpu_enabled, backend_dependencies=backend_dependencies
)
"""

if "fastertransformer" in backends:
be = "fastertransformer"
Expand Down Expand Up @@ -1433,9 +1593,14 @@ def create_build_dockerfiles(
)
dockerfileargmap["GPU_BASE_IMAGE"] = gpu_base_image

create_dockerfile_buildbase(
FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap
)
if target_platform() == "rhel":
create_dockerfile_buildbase_rhel(
FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap
)
else:
create_dockerfile_buildbase(
FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap
)

if target_platform() == "windows":
create_dockerfile_windows(
Expand Down Expand Up @@ -1651,6 +1816,17 @@ def core_build(
os.path.join(repo_install_dir, "lib", "tritonserver.lib"),
os.path.join(install_dir, "bin"),
)
elif target_platform() == "rhel":
cmake_script.mkdir(os.path.join(install_dir, "bin"))
cmake_script.cp(
os.path.join(repo_install_dir, "bin", "tritonserver"),
os.path.join(install_dir, "bin"),
)
cmake_script.mkdir(os.path.join(install_dir, "lib64"))
cmake_script.cp(
os.path.join(repo_install_dir, "lib64", "libtritonserver.so"),
os.path.join(install_dir, "lib64"),
)
else:
cmake_script.mkdir(os.path.join(install_dir, "bin"))
cmake_script.cp(
Expand Down Expand Up @@ -2128,7 +2304,7 @@ def enable_all():
"--target-platform",
required=False,
default=None,
help='Target platform for build, can be "linux", "windows" or "igpu". If not specified, build targets the current platform.',
help='Target platform for build, can be "linux", "rhel", "windows" or "igpu". If not specified, build targets the current platform.',
)
parser.add_argument(
"--target-machine",
Expand Down
22 changes: 16 additions & 6 deletions qa/L0_infer/install_and_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,24 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Determine the operating system to call the correct package manager.
ID_LIKE=$(grep -Po '(?<=ID_LIKE=).*' /etc/os-release | awk -F= '{print $1}' | tr -d '"' | awk '{print $1}')

# Note: This script is to be used with customized triton containers that need
# dependencies to run L0_infer tests
apt-get update && \
apt-get install -y --no-install-recommends \
curl \
jq \
python3 \
python3-pip
if [[ "$ID_LIKE" =~ "debian" ]]; then
apt-get update && \
apt-get install -y --no-install-recommends \
curl \
jq \
python3 \
python3-pip
else
yum install -y \
jq \
curl
fi

pip3 install --upgrade pip
# install client libraries
pip3 install tritonclient[all]
Expand Down
11 changes: 10 additions & 1 deletion src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,14 +138,23 @@ else()
)
endif()

set(LIB_DIR "lib")
if(LINUX)
file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE")
if(${DISTRO_ID_LIKE} MATCHES "rhel|centos")
set (LIB_DIR "lib64")
endif(${DISTRO_ID_LIKE} MATCHES "rhel|centos")
endif(LINUX)
set(TRITON_CORE_HEADERS_ONLY OFF)

set_target_properties(
main
PROPERTIES
POSITION_INDEPENDENT_CODE ON
SKIP_BUILD_RPATH TRUE
BUILD_WITH_INSTALL_RPATH TRUE
INSTALL_RPATH_USE_LINK_PATH FALSE
INSTALL_RPATH "$\{ORIGIN\}/../lib"
INSTALL_RPATH "$\{ORIGIN\}/../${LIB_DIR}"
)

target_link_libraries(
Expand Down

0 comments on commit 66912aa

Please sign in to comment.