Update to JavaCPP 1.4 and add support for CUDA 9.1 (#2555)

* Update to JavaCPP 1.4 and add support for CUDA 9.1

* Remove dependency on MKL classifier not available for all platforms

* Fix calls to change-cuda-versions.sh and change-scala-versions.sh

.appveyor.yml

@@ -16,6 +16,8 @@ environment:
       SCALA: 2.10
     - CUDA: 9.0
       SCALA: 2.11
+    - CUDA: 9.1
+      SCALA: 2.11
 
 init:
   - wmic computersystem set AutomaticManagedPagefile=False

.travis.yml

@@ -37,6 +37,10 @@ matrix:
       env: OS=linux-x86_64 CUDA=9.0 SCALA=2.11
       install: true
       script: bash ./ci/build-linux-x86_64.sh
+    - os: linux
+      env: OS=linux-x86_64 CUDA=9.1 SCALA=2.11
+      install: true
+      script: bash ./ci/build-linux-x86_64.sh
     - os: osx
       osx_image: xcode7.3
       env: OS=macosx-x86_64 EXT=avx2 SCALA=2.10
@@ -57,4 +61,9 @@ matrix:
       env: OS=macosx-x86_64 CUDA=9.0 SCALA=2.11
       install: true
       script: bash ./ci/build-macosx-x86_64.sh
+    - os: osx
+      osx_image: xcode8.3
+      env: OS=macosx-x86_64 CUDA=9.1 SCALA=2.11
+      install: true
+      script: bash ./ci/build-macosx-x86_64.sh
 

change-cuda-versions.sh

@@ -21,11 +21,14 @@
 
 set -e
 
-VALID_VERSIONS=( 8.0 9.0 )
+VALID_VERSIONS=( 8.0 9.0 9.1 )
 CUDA_80_VERSION="8\.0"
 CUDA_90_VERSION="9\.0"
+CUDA_91_VERSION="9\.1"
 CUDNN_60_VERSION="6\.0"
 CUDNN_70_VERSION="7\.0"
+JAVACPP_13_VERSION="1\.3"
+JAVACPP_14_VERSION="1\.4"
 
 usage() {
   echo "Usage: $(basename $0) [-h|--help] <cuda version to be used>
@@ -40,7 +43,7 @@ if [[ ($# -ne 1) || ( $1 == "--help") ||  $1 == "-h" ]]; then
   usage
 fi
 
-TO_VERSION=$1
+VERSION=$1
 
 check_cuda_version() {
   for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
@@ -49,23 +52,25 @@ check_cuda_version() {
 }
 
 
-check_cuda_version "$TO_VERSION"
-
-if [ $TO_VERSION = "9.0" ]; then
-  FROM_BINARY="-8\.0"
-  TO_BINARY="-9\.0"
-  FROM_VERSION=$CUDA_80_VERSION
-  TO_VERSION=$CUDA_90_VERSION
-  FROM_VERSION2=$CUDNN_60_VERSION
-  TO_VERSION2=$CUDNN_70_VERSION
-else
-  FROM_BINARY="-9\.0"
-  TO_BINARY="-8\.0"
-  FROM_VERSION=$CUDA_90_VERSION
-  TO_VERSION=$CUDA_80_VERSION
-  FROM_VERSION2=$CUDNN_70_VERSION
-  TO_VERSION2=$CUDNN_60_VERSION
-fi
+check_cuda_version "$VERSION"
+
+case $VERSION in
+  9.1)
+    VERSION=$CUDA_91_VERSION
+    VERSION2=$CUDNN_70_VERSION
+    VERSION3=$JAVACPP_14_VERSION
+    ;;
+  9.0)
+    VERSION=$CUDA_90_VERSION
+    VERSION2=$CUDNN_70_VERSION
+    VERSION3=$JAVACPP_13_VERSION
+    ;;
+  8.0)
+    VERSION=$CUDA_80_VERSION
+    VERSION2=$CUDNN_60_VERSION
+    VERSION3=$JAVACPP_13_VERSION
+    ;;
+esac
 
 sed_i() {
   sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
@@ -77,20 +82,24 @@ echo "Updating CUDA versions in pom.xml files to CUDA $1";
 
 BASEDIR=$(dirname $0)
 
-#Artifact ids, ending with "-8.0" or "-9.0". nd4j-cuda, deeplearning4j-cuda, etc.
+#Artifact ids, ending with "-8.0", "-9.0", etc. nd4j-cuda, etc.
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
-  -exec bash -c "sed_i 's/\(artifactId>.*\)'$FROM_BINARY'<\/artifactId>/\1'$TO_BINARY'<\/artifactId>/g' {}" \;
+  -exec bash -c "sed_i 's/\(artifactId>nd4j-cuda-\)...<\/artifactId>/\1'$VERSION'<\/artifactId>/g' {}" \;
 
-#Artifact ids, ending with "-8.0-platform" or "-9.0-platform". nd4j-cuda-platform, etc.
+#Artifact ids, ending with "-8.0-platform", "-9.0-platform", etc. nd4j-cuda-platform, etc.
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
-  -exec bash -c "sed_i 's/\(artifactId>.*\)'$FROM_BINARY'-platform<\/artifactId>/\1'$TO_BINARY'-platform<\/artifactId>/g' {}" \;
+  -exec bash -c "sed_i 's/\(artifactId>nd4j-cuda-\)...-platform<\/artifactId>/\1'$VERSION'-platform<\/artifactId>/g' {}" \;
 
-#CUDA versions, like <cuda.version>9.0</cuda.version>
+#CUDA versions, like <cuda.version>9.1</cuda.version>
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
-  -exec bash -c "sed_i 's/\(cuda.version>\)'$FROM_VERSION'<\/cuda.version>/\1'$TO_VERSION'<\/cuda.version>/g' {}" \;
+  -exec bash -c "sed_i 's/\(cuda.version>\)...<\/cuda.version>/\1'$VERSION'<\/cuda.version>/g' {}" \;
 
 #cuDNN versions, like <cudnn.version>7.0</cudnn.version>
 find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
-  -exec bash -c "sed_i 's/\(cudnn.version>\)'$FROM_VERSION2'<\/cudnn.version>/\1'$TO_VERSION2'<\/cudnn.version>/g' {}" \;
+  -exec bash -c "sed_i 's/\(cudnn.version>\)...<\/cudnn.version>/\1'$VERSION2'<\/cudnn.version>/g' {}" \;
+
+#JavaCPP versions, like <javacpp-presets.cuda.version>1.4</javacpp-presets.cuda.version>
+find "$BASEDIR" -name 'pom.xml' -not -path '*target*' \
+  -exec bash -c "sed_i 's/\(javacpp-presets.cuda.version>\)...<\/javacpp-presets.cuda.version>/\1'$VERSION3'<\/javacpp-presets.cuda.version>/g' {}" \;
 
 echo "Done updating CUDA versions.";

ci/build-android.sh

@@ -25,7 +25,7 @@ cd $TRAVIS_BUILD_DIR/../libnd4j/
 sed -i /cmake_minimum_required/d CMakeLists.txt
 MAKEJ=2 bash buildnativeoperations.sh -platform $OS
 cd $TRAVIS_BUILD_DIR/
-source change-scala-versions.sh $SCALA
+bash change-scala-versions.sh $SCALA
 mvn clean $MAVEN_PHASE -B -U --settings ./ci/settings.xml -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlocal.software.repository=sonatype \
     -Djavacpp.platform=$OS -pl '!nd4j-backends/nd4j-backend-impls/nd4j-cuda,!nd4j-backends/nd4j-backend-impls/nd4j-cuda-platform,!nd4j-backends/nd4j-tests'
 

ci/build-linux-x86_64.sh

@@ -51,12 +51,12 @@ docker run -ti -e SONATYPE_USERNAME -e SONATYPE_PASSWORD -v $HOME/.m2:/root/.m2
     fi; \
     cd /build/nd4j/; \
     if [[ -n \"${CUDA:-}\" ]]; then \
-        source change-cuda-versions.sh $CUDA; \
+        bash change-cuda-versions.sh $CUDA; \
         EXTRA_OPTIONS=; \
     else \
         EXTRA_OPTIONS='-pl !nd4j-backends/nd4j-backend-impls/nd4j-cuda,!nd4j-backends/nd4j-backend-impls/nd4j-cuda-platform,!nd4j-backends/nd4j-tests'; \
     fi; \
-    source change-scala-versions.sh $SCALA; \
+    bash change-scala-versions.sh $SCALA; \
     mvn clean $MAVEN_PHASE -B -U --settings ./ci/settings.xml -Dmaven.test.skip=true -Dlocal.software.repository=sonatype \
         -Djavacpp.extension=${EXT:-} \$EXTRA_OPTIONS -DprotocCommand=/build/protobuf-$PROTOBUF/src/protoc;"
 

ci/build-macosx-x86_64.sh

@@ -29,6 +29,8 @@ if [[ "${CUDA:-}" == "8.0" ]]; then
     CUDA_URL=https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_mac-dmg
 elif [[ "${CUDA:-}" == "9.0" ]]; then
     CUDA_URL=https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda_9.0.176_mac-dmg
+elif [[ "${CUDA:-}" == "9.1" ]]; then
+    CUDA_URL=https://developer.nvidia.com/compute/cuda/9.1/Prod/local_installers/cuda_9.1.85_mac
 fi
 if [[ -n ${CUDA_URL:-} ]]; then
     curl --retry 10 -L -o $HOME/cuda.dmg $CUDA_URL
@@ -45,12 +47,12 @@ if [[ -n "${CUDA:-}" ]]; then
 fi
 cd $TRAVIS_BUILD_DIR/
 if [[ -n "${CUDA:-}" ]]; then
-    source change-cuda-versions.sh $CUDA
+    bash change-cuda-versions.sh $CUDA
     EXTRA_OPTIONS=
 else
     EXTRA_OPTIONS='-pl !nd4j-backends/nd4j-backend-impls/nd4j-cuda,!nd4j-backends/nd4j-backend-impls/nd4j-cuda-platform,!nd4j-backends/nd4j-tests'
 fi
-source change-scala-versions.sh $SCALA
+bash change-scala-versions.sh $SCALA
 mvn clean $MAVEN_PHASE -B -U --settings ./ci/settings.xml -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlocal.software.repository=sonatype \
     -Djavacpp.extension=${EXT:-} $EXTRA_OPTIONS
 

ci/build-windows-x86_64.cmd

@@ -20,6 +20,9 @@ if "%CUDA%" == "8.0" (
 if "%CUDA%" == "9.0" (
     set "CUDA_URL=https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda_9.0.176_windows-exe"
 )
+if "%CUDA%" == "9.1" (
+    set "CUDA_URL=https://developer.nvidia.com/compute/cuda/9.1/Prod/local_installers/cuda_9.1.85_windows"
+)
 if not "%CUDA_URL%" == "" (
     curl --retry 10 -L -o cuda.exe %CUDA_URL%
     cuda.exe -s
@@ -36,11 +39,11 @@ bash -lc "pacman -S --needed --noconfirm base-devel make mingw-w64-x86_64-cmake
 bash -c "cd ../libnd4j/; MAKEJ=2 bash buildnativeoperations.sh -c cpu -e $EXT"
 if not "%CUDA%" == "" (
     bash -c "cd ../libnd4j/; MAKEJ=1 bash buildnativeoperations.sh -c cuda -v $CUDA -cc 30"
-    bash -c "source change-cuda-versions.sh $CUDA"
+    bash -c "change-cuda-versions.sh $CUDA"
     set "EXTRA_OPTIONS="
 ) else (
     set "EXTRA_OPTIONS=-pl !nd4j-backends/nd4j-backend-impls/nd4j-cuda,!nd4j-backends/nd4j-backend-impls/nd4j-cuda-platform,!nd4j-backends/nd4j-tests"
 )
-bash -c "source change-scala-versions.sh $SCALA"
+bash -c "change-scala-versions.sh $SCALA"
 call mvn clean %MAVEN_PHASE% -B -U --settings .\ci\settings.xml -Dmaven.test.skip=true -Dlocal.software.repository=sonatype ^
     -Djavacpp.extension=%EXT% %EXTRA_OPTIONS%

nd4j-backends/nd4j-backend-impls/nd4j-cuda-platform/pom.xml

@@ -12,14 +12,15 @@
     <properties>
         <cuda.version>8.0</cuda.version>
         <cudnn.version>6.0</cudnn.version>
+        <javacpp-presets.cuda.version>1.3</javacpp-presets.cuda.version>
         <nd4j.backend>nd4j-cuda-${cuda.version}</nd4j.backend>
     </properties>
 
     <dependencies>
         <dependency>
             <groupId>org.bytedeco.javacpp-presets</groupId>
             <artifactId>cuda-platform</artifactId>
-            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.version}</version>
+            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.cuda.version}</version>
         </dependency>
 
         <dependency>

nd4j-backends/nd4j-backend-impls/nd4j-cuda/pom.xml

@@ -12,6 +12,7 @@
     <properties>
         <cuda.version>8.0</cuda.version>
         <cudnn.version>6.0</cudnn.version>
+        <javacpp-presets.cuda.version>1.3</javacpp-presets.cuda.version>
         <maven.compiler.target>1.7</maven.compiler.target>
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.testTarget>1.8</maven.compiler.testTarget>
@@ -198,12 +199,12 @@
         <dependency>
             <groupId>org.bytedeco.javacpp-presets</groupId>
             <artifactId>cuda</artifactId>
-            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.version}</version>
+            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.cuda.version}</version>
         </dependency>
         <dependency>
             <groupId>org.bytedeco.javacpp-presets</groupId>
             <artifactId>cuda</artifactId>
-            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.version}</version>
+            <version>${cuda.version}-${cudnn.version}-${javacpp-presets.cuda.version}</version>
             <classifier>${dependency.platform}</classifier>
         </dependency>
 

nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java

@@ -1,4 +1,4 @@
-// Targeted by JavaCPP version 1.3.4-SNAPSHOT: DO NOT EDIT THIS FILE
+// Targeted by JavaCPP version 1.4: DO NOT EDIT THIS FILE
 
 package org.nd4j.nativeblas;
 

nd4j-backends/nd4j-backend-impls/nd4j-native-platform/pom.xml

@@ -11,7 +11,7 @@
 
     <properties>
         <openblas.version>0.2.20</openblas.version>
-        <mkl.version>2017.3</mkl.version>
+        <mkl.version>2018.1</mkl.version>
         <nd4j.backend>nd4j-native</nd4j.backend>
     </properties>
 

nd4j-backends/nd4j-backend-impls/nd4j-native/pom.xml

@@ -11,7 +11,7 @@
 
     <properties>
         <openblas.version>0.2.20</openblas.version>
-        <mkl.version>2017.3</mkl.version>
+        <mkl.version>2018.1</mkl.version>
         <maven.compiler.target>1.7</maven.compiler.target>
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.testTarget>1.8</maven.compiler.testTarget>
@@ -55,12 +55,12 @@
             <artifactId>mkl</artifactId>
             <version>${mkl.version}-${javacpp-presets.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.bytedeco.javacpp-presets</groupId>
-            <artifactId>mkl</artifactId>
-            <version>${mkl.version}-${javacpp-presets.version}</version>
-            <classifier>${dependency.platform}</classifier>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.bytedeco.javacpp-presets</groupId>-->
+<!--            <artifactId>mkl</artifactId>-->
+<!--            <version>${mkl.version}-${javacpp-presets.version}</version>-->
+<!--            <classifier>${dependency.platform}</classifier>-->
+<!--        </dependency>-->
 
         <dependency>
             <groupId>org.nd4j</groupId>