Create and upload binaries to the release tag through an automated sc…

…ript

Signed-off-by: shams <shamsazeem20@gmail.com>

build_jar.py

@@ -119,8 +119,9 @@ def get_platform():
         elif args.cuda_version == "10.1":
             command.append("-Dcuda.javacpp.version=10.1-7.6-1.5.2")
 
-    for option in args.options:
-        command.append("-D{}".format(option))
+    if args.options:
+        for option in args.options:
+            command.append("-D{}".format(option))
 
     command.append("-Denforcer.skip=true")
 

create-all-binaries.sh

@@ -1,15 +1,23 @@
 #!/bin/bash
 
-set -euxo pipefail
+set -euo pipefail
 
 declare -a platforms=("windows-x86_64" "macosx-x86_64" "linux-x86_64")
-declare -a chips=("gpu")
+declare -a chips=("cpu" "gpu")
 declare -a spins=("minimal" "pmml" "python" "all")
 
 mkdir -p builds
 
 PROJECT_VERSION="$(mvn help:evaluate -D expression=project.version | grep -e '^[^\[]')"
+LOGS_FILE=builds/mavenlogs.log
 
+if [[ -f "${LOGS_FILE}" ]]; then
+    rm "${LOGS_FILE}"
+fi
+
+echo "You can find maven build logs at: $(realpath "${LOGS_FILE}")"
+
+INDEX=0
 ## now loop through the above array
 for platform in "${platforms[@]}"
 do
@@ -22,20 +30,42 @@ do
       fi
 
       if [[ "${chip}" == 'gpu' ]]; then
-        declare -a cuda_versions=("10.1" "10.2")
+        declare -a cuda_versions=("10.0" "10.1" "10.2")
 
         for cuda_version in "${cuda_versions[@]}"
         do
-          echo "Compiling for ${platform} | gpu | ${cuda_version} | ${spin}"
+          echo "-------------------------------------------------------"
+
+          INDEX=$((INDEX + 1))
+
+          OUTPUT_FILE_NAME=konduit-serving-uberjar-"${PROJECT_VERSION}"-"${spin}"-"${platform}"-"${chip}"-cuda-"${cuda_version}".jar
+          TARGET=builds/"${OUTPUT_FILE_NAME}"
+          COMMAND="python build_jar.py -p ""${platform}"" -c ""${chip}"" -cv ""${cuda_version}"" -s ""${spin}"" --target ""${TARGET}"""
+
+          echo "${INDEX}: Compiling for ${platform} | gpu | ${cuda_version} | ${spin} ==> Command: $(${COMMAND} -sbc)"
+          bash -c "${COMMAND}" >> "${LOGS_FILE}"
 
-          python build_jar.py -p "${platform}" -c "${chip}" -cv ${cuda_version} -s "${spin}" \
-              --target builds/konduit-serving-uberjar-"${PROJECT_VERSION}"-"${spin}"-"${platform}"-"${chip}"-${cuda_version}.jar
+          echo "==> Binary created at: $(realpath "${TARGET}")"
+          echo "STATS: $(stat -c "%s %y" "${TARGET}" | awk '{ byte=$1 /1024/1024; print "("byte "M | Date: " $2 " | Time: " $3 " "  $4")"}')"
+
+          ./upload-binary.sh "${OUTPUT_FILE_NAME}"
         done
       else
-        echo "Compiling for ${platform} | ${chip} | ${spin}"
+        echo "-------------------------------------------------------"
+
+        INDEX=$((INDEX + 1))
+
+        OUTPUT_FILE_NAME=konduit-serving-uberjar-"${PROJECT_VERSION}"-"${spin}"-"${platform}"-"${chip}".jar
+        TARGET=builds/"${OUTPUT_FILE_NAME}"
+        COMMAND="python build_jar.py -p ""${platform}"" -c ""${chip}"" -s ""${spin}"" --target ""${TARGET}"""
+
+        echo "${INDEX}: Compiling for ${platform} | ${chip} | ${spin} ==> Command: $(${COMMAND} -sbc)"
+        bash -c "${COMMAND}" >> "${LOGS_FILE}"
+
+        echo "==> Binary created at: $(realpath "${TARGET}")"
+        echo "STATS: $(stat -c "%s %y" "${TARGET}" | awk '{ byte=$1 /1024/1024; print "("byte "M | Date: " $2 " | Time: " $3 " "  $4")"}')"
 
-        python build_jar.py -p "${platform}" -c "${chip}" -s "${spin}" \
-            --target builds/konduit-serving-uberjar-"${PROJECT_VERSION}"-"${spin}"-"${platform}"-"${chip}".jar
+        ./upload-binary.sh "${OUTPUT_FILE_NAME}"
       fi
     done
   done

pom.xml

@@ -94,7 +94,7 @@
         <micrometer.version>1.2.0</micrometer.version>
         <netty.version>4.1.48.Final</netty.version>
         <arrow.version>0.15.1</arrow.version>
-        <dl4j.version>1.0.0-SNAPSHOT</dl4j.version>
+        <dl4j.version>1.0.0-beta7</dl4j.version>
         <datavec.version>${dl4j.version}</datavec.version>
         <nd4j.version>${dl4j.version}</nd4j.version>
         <slf4j.version>1.7.26</slf4j.version>
@@ -604,10 +604,8 @@
             <id>ossrh</id>
             <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
         </repository>
-
     </distributionManagement>
 
-
     <build>
         <extensions>
             <extension>
@@ -618,7 +616,6 @@
         </extensions>
 
         <plugins>
-
             <plugin>
                 <groupId>kr.motd.maven</groupId>
                 <artifactId>os-maven-plugin</artifactId>
@@ -632,7 +629,6 @@
                     </execution>
                 </executions>
             </plugin>
-
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-enforcer-plugin</artifactId>
@@ -733,6 +729,11 @@
             <id>konduit-packages</id>
             <url>https://packages.konduit.ai/repository/maven-public/</url>
         </repository>
+        <repository>
+            <id>sonatype-nexus-staging</id>
+            <name>Sonatype Nexus Staging</name>
+            <url>https://oss.sonatype.org/content/repositories/orgdeeplearning4j-1115/</url>
+        </repository>
         <repository>
             <id>snapshots-repo</id>
             <url>https://oss.sonatype.org/content/repositories/snapshots</url>

python/cli_init.py

@@ -172,7 +172,7 @@ def get_jar_url(platform, version, spin, chip, cuda_version=None):
                                          tag=DOWNLOAD_TAG,
                                          spin=spin,
                                          chip=chip,
-                                         cuda_version=("-" + cuda_version) if chip == "gpu" else "")
+                                         cuda_version=("-cuda" + cuda_version) if chip == "gpu" else "")
 
 
 git_tags = list(set(get_git_tags()).difference(INCOMPATIBLE_COMPILATION_TAGS))

upload-binary.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+BINARY_FILE_NAME=$1
+
+echo "---"
+echo "Looking for old asset file on repo..."
+RELEASE_DATA=$(curl -s -X GET https://api.github.com/repos/KonduitAI/konduit-serving/releases/25710929)
+
+id=" "
+eval "$(echo "${RELEASE_DATA}" | grep -C2 "name.:.\+""${BINARY_FILE_NAME}""" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=')"
+echo "---"
+if [ "${id}" = " " ]; then
+    echo "No old asset file found with the same file name."
+else
+    echo "Deleting old asset file..."
+    curl -s -u "$(cat ~/TOKEN)" -X "DELETE" "https://api.github.com/repos/KonduitAI/konduit-serving/releases/assets/""${id}"""
+    echo "Old asset file deleted..."
+fi
+
+echo "---"
+echo "Uploading ${BINARY_FILE_NAME}"
+curl -u "$(cat ~/TOKEN)" --data-binary @builds/""${BINARY_FILE_NAME}"" -H "Content-Type:application/octet-stream" -X POST "https://uploads.github.com/repos/KonduitAI/konduit-serving/releases/25710929/assets?name=""${BINARY_FILE_NAME}"""
+echo "---"
+echo "Uploaded ""${BINARY_FILE_NAME}"" at URL: ${UPLOAD_URL}"