-
Notifications
You must be signed in to change notification settings - Fork 76
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #244 from clara-genomics/dev-v0.4.0
Merge dev-v0.4.0 into master
- Loading branch information
Showing
105 changed files
with
4,612 additions
and
3,951 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,6 +56,6 @@ if [ "${CUDA:0:2}" == '10' ]; then | |
| fi | ||
|
|
||
| # Cleanup local git | ||
| cd $1 | ||
| cd "$1" | ||
| git clean -xdf | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. | ||
| # | ||
| # NVIDIA CORPORATION and its licensors retain all intellectual property | ||
| # and proprietary rights in and to this software, related documentation | ||
| # and any modifications thereto. Any use, reproduction, disclosure or | ||
| # distribution of this software and related documentation without an express | ||
| # license agreement from NVIDIA CORPORATION is strictly prohibited. | ||
| # | ||
|
|
||
| DOCKER_IMAGE="gpuci/clara-genomics-base:cuda10.0-ubuntu16.04-gcc5-py3.7" | ||
| REPO_PATH=${PWD} | ||
| RAPIDS_DIR_IN_CONTAINER="/rapids" | ||
| CPP_BUILD_DIR="cpp" | ||
| CONTAINER_SHELL_ONLY=0 | ||
|
|
||
| SHORTHELP="$(basename "$0") [-h] [-H] [-s] [-r <repo_dir>] [-i <image_name>]" | ||
| LONGHELP="${SHORTHELP} | ||
| Build and test your local repository using a base gpuCI Docker image | ||
| where: | ||
| -H Show this help text | ||
| -r Path to repository (defaults to working directory) | ||
| -i Use Docker image (default is ${DOCKER_IMAGE}) | ||
| -s Skip building and testing and start an interactive shell in a container of the Docker image | ||
| " | ||
|
|
||
| # Limit GPUs available to container based on CUDA_VISIBLE_DEVICES | ||
| if [[ -z "${CUDA_VISIBLE_DEVICES}" ]]; then | ||
| NVIDIA_VISIBLE_DEVICES="all" | ||
| else | ||
| NVIDIA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} | ||
| fi | ||
|
|
||
| while getopts ":hHr:i:s" option; do | ||
| case ${option} in | ||
| r) | ||
| REPO_PATH=${OPTARG} | ||
| ;; | ||
| i) | ||
| DOCKER_IMAGE=${OPTARG} | ||
| ;; | ||
| s) | ||
| CONTAINER_SHELL_ONLY=1 | ||
| ;; | ||
| h) | ||
| echo "${SHORTHELP}" | ||
| exit 0 | ||
| ;; | ||
| H) | ||
| echo "${LONGHELP}" | ||
| exit 0 | ||
| ;; | ||
| *) | ||
| echo "ERROR: Invalid flag" | ||
| echo "${SHORTHELP}" | ||
| exit 1 | ||
| ;; | ||
| esac | ||
| done | ||
| IMAGE_FOLDER_NAME="build_$(echo $(basename "${DOCKER_IMAGE}")|sed -e 's/:/_/g')" | ||
| REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")" | ||
| CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${IMAGE_FOLDER_NAME}/${CPP_BUILD_DIR}" | ||
|
|
||
|
|
||
| # BASE_CONTAINER_BUILD_DIR is named after the image name, allowing for | ||
| # multiple image builds to coexist on the local filesystem. This will | ||
| # be mapped to the typical BUILD_DIR inside of the container. Builds | ||
| # running in the container generate build artifacts just as they would | ||
| # in a bare-metal environment, and the host filesystem is able to | ||
| # maintain the host build in BUILD_DIR as well. | ||
| # shellcheck disable=SC2001,SC2005,SC2046 | ||
| BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/${IMAGE_FOLDER_NAME} | ||
| CPP_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/cpp | ||
|
|
||
|
|
||
| BUILD_SCRIPT="#!/bin/bash | ||
| set -e | ||
| WORKSPACE=${REPO_PATH_IN_CONTAINER} | ||
| PREBUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/prebuild.sh | ||
| BUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/build.sh | ||
| if [ -f \${PREBUILD_SCRIPT} ]; then | ||
| source \${PREBUILD_SCRIPT} | ||
| fi | ||
| yes | source \${BUILD_SCRIPT} | ||
| " | ||
|
|
||
| if (( CONTAINER_SHELL_ONLY == 0 )); then | ||
| COMMAND="${CPP_BUILD_DIR_IN_CONTAINER}/build.sh || bash" | ||
| else | ||
| COMMAND="bash" | ||
| fi | ||
|
|
||
| # Create the build dir for the container to mount, generate the build script inside of it | ||
| mkdir -p "${BASE_CONTAINER_BUILD_DIR}" | ||
| mkdir -p "${CPP_CONTAINER_BUILD_DIR}" | ||
| # Create build directories. This is to ensure correct owner for directories. If | ||
| # directories don't exist there is side effect from docker volume mounting creating build | ||
| # directories owned by root(volume mount point(s)) | ||
|
|
||
| echo "${BUILD_SCRIPT}" > "${CPP_CONTAINER_BUILD_DIR}/build.sh" | ||
| chmod ugo+x "${CPP_CONTAINER_BUILD_DIR}/build.sh" | ||
|
|
||
| # Mount passwd and group files to docker. This allows docker to resolve username and group | ||
| # avoiding these nags: | ||
| # * groups: cannot find name for group ID ID | ||
| # * I have no name!@id:/$ | ||
| # For ldap user user information is not present in system /etc/passwd and /etc/group files. | ||
| # Hence we generate dummy files for ldap users which docker uses to resolve username and group | ||
|
|
||
| PASSWD_FILE="/etc/passwd" | ||
| GROUP_FILE="/etc/group" | ||
|
|
||
| USER_FOUND=$(grep -wc "$(whoami)" < "$PASSWD_FILE") | ||
| if [ "$USER_FOUND" == 0 ]; then | ||
| echo "Local User not found, LDAP WAR for docker mounts activated. Creating dummy passwd and group" | ||
| echo "files to allow docker resolve username and group" | ||
| cp "$PASSWD_FILE" /tmp/passwd | ||
| PASSWD_FILE="/tmp/passwd" | ||
| cp "$GROUP_FILE" /tmp/group | ||
| GROUP_FILE="/tmp/group" | ||
| echo "$(whoami):x:$(id -u):$(id -g):$(whoami),,,:$HOME:$SHELL" >> "$PASSWD_FILE" | ||
| echo "$(whoami):x:$(id -g):" >> "$GROUP_FILE" | ||
| fi | ||
|
|
||
| # Run the generated build script in a container | ||
| sudo docker pull "${DOCKER_IMAGE}" | ||
| sudo docker run --runtime=nvidia --rm -it -e NVIDIA_VISIBLE_DEVICES="${NVIDIA_VISIBLE_DEVICES}" \ | ||
| -u "$(id -u)":"$(id -g)" \ | ||
| -v "${REPO_PATH}":"${REPO_PATH_IN_CONTAINER}" \ | ||
| -v "$PASSWD_FILE":/etc/passwd:ro \ | ||
| -v "$GROUP_FILE":/etc/group:ro \ | ||
| --cap-add=SYS_PTRACE \ | ||
| "${DOCKER_IMAGE}" bash -c "${COMMAND}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # | ||
| # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. | ||
| # | ||
| # NVIDIA CORPORATION and its licensors retain all intellectual property | ||
| # and proprietary rights in and to this software, related documentation | ||
| # and any modifications thereto. Any use, reproduction, disclosure or | ||
| # distribution of this software and related documentation without an express | ||
| # license agreement from NVIDIA CORPORATION is strictly prohibited. | ||
| # | ||
|
|
||
| function(validate_boolean CMAKE_OPTION) | ||
| if ((NOT ${CMAKE_OPTION} STREQUAL "ON") AND (NOT ${CMAKE_OPTION} STREQUAL "OFF")) | ||
| message(FATAL_ERROR "${CMAKE_OPTION} can only be set to ON/OFF") | ||
| endif() | ||
| endfunction(validate_boolean) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.