Merge branch 'release/1.0.13'

BD2KGenomics · May 11, 2017 · 38c4707 · 38c4707
2 parents 013c534 + 7c6e7ed
commit 38c4707
Show file tree

Hide file tree

Showing 15 changed files with 1,654 additions and 1,214 deletions.
diff --git a/.dockstore/self-installs/dockstore-client-1.0.1.jar b/.dockstore/self-installs/dockstore-client-1.0.1.jar
diff --git a/.dockstore/self-installs/dockstore-client-1.2.1.jar b/.dockstore/self-installs/dockstore-client-1.2.1.jar
diff --git a/Dockerfile b/Dockerfile
@@ -32,8 +32,8 @@ RUN pip install jsonschema jsonmerge openpyxl sets json-spec elasticsearch semve
 
 #install cwltool in the container
 #use the version required by dockstore
-RUN pip install setuptools==24.0.3
-RUN pip install cwl-runner cwltool==1.0.20160712154127 schema-salad==1.14.20160708181155 avro==1.8.1
+RUN pip install setuptools==28.8.0
+RUN pip install cwl-runner cwltool==1.0.20170217172322 schema-salad==2.2.20170222151604 avro==1.8.1
 RUN pip install functools32==3.2.3.post2
 
 #Patch the cwltool code that sets up the docker run command line
@@ -50,8 +50,6 @@ RUN pip install functools32==3.2.3.post2
 #of python is the right one?
 COPY job.patch /usr/local/lib/python2.7/dist-packages/cwltool/job.patch
 RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/job.patch
-COPY main.patch /usr/local/lib/python2.7/dist-packages/cwltool/main.patch
-RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/main.patch
 COPY process.patch /usr/local/lib/python2.7/dist-packages/cwltool/process.patch
 RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/process.patch
 

diff --git a/Dockstore.cwl b/Dockstore.cwl
@@ -24,7 +24,7 @@ dct:creator:
 
 requirements:
   - class: DockerRequirement
-    dockerPull: "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.12"
+    dockerPull: "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13"
 hints:
   - class: ResourceRequirement
     coresMin: 1

diff --git a/Dockstore/dockstore b/Dockstore/dockstore
@@ -5,11 +5,14 @@ set -o errexit
 # somewhere on your $PATH, like ~/bin. The rest of SeqWare will be
 # installed upon first run into the ~/.dockstore directory.
 
-DEFAULT_DOCKSTORE_VERSION="1.0.1"
+# Encase variables inside brackets to prevent issues.
+# $DOCKSTORE_VERSION would cause problems if $DOCKSTORE exists.  ${DOCKSTORE_VERSION} would be better.
+
+DEFAULT_DOCKSTORE_VERSION="1.2.1"
 
 export DOCKSTORE_VERSION="${DOCKSTORE_VERSION:-"$DEFAULT_DOCKSTORE_VERSION"}"
 
-case $DOCKSTORE_VERSION in
+case ${DOCKSTORE_VERSION} in
     *SNAPSHOT) SNAPSHOT="YES" ;;
     *) SNAPSHOT="NO" ;;
 esac
@@ -36,17 +39,20 @@ fi
 export DOCKSTORE_HOME="${DOCKSTORE_HOME:-"$HOME/.dockstore"}"
 # export DOCKSTORE_HOME="${DOCKSTORE_HOME:-"$HOME/dockstore"}"
 
-if $cygwin; then
+if ${cygwin}; then
     export DOCKSTORE_HOME=`cygpath -w "$DOCKSTORE_HOME"`
 fi
 
 export DOCKSTORE_JAR="$DOCKSTORE_HOME/self-installs/dockstore-client-$DOCKSTORE_VERSION.jar"
-#export DOCKSTORE_JAR="$DOCKSTORE_HOME/dockstore-client/target/dockstore-client-$DOCKSTORE_VERSION.jar"
 export DOCKSTORE_SETTINGS="${DOCKSTORE_SETTINGS:-"$DOCKSTORE_HOME/config"}"
 
 # normalize $0 on certain BSDs
 if [ "$(dirname "$0")" = "." ]; then
-    SCRIPT="$(which $(basename "$0"))"
+    if ! SCRIPT="$(which $(basename "$0"))"; then
+         if [[ "$@" != *"--script"* ]]; then
+            echo "dockstore needs to be on PATH for certain commands to work (like --upgrade)"
+        fi
+    fi
 else
     SCRIPT="$0"
 fi
@@ -64,7 +70,7 @@ done
 
 function make_native_path {
     # ensure we have native paths
-    if $cygwin && [[ "$1"  == /* ]]; then
+    if ${cygwin} && [[ "$1"  == /* ]]; then
         echo -n "$(cygpath -wp "$1")"
     elif [[ "$OSTYPE" == "msys" && "$1"  == /* ]]; then
         echo -n "$(sh -c "(cd $1 2</dev/null && pwd -W) || echo $1 | sed 's/^\\/\([a-z]\)/\\1:/g'")"
@@ -119,13 +125,13 @@ export JVM_OPTS="${JVM_OPTS:-"$JAVA_OPTS"}"
 
 # Handle jline issue with cygwin not propagating OSTYPE through java subprocesses: https://github.com/jline/jline2/issues/62
 cygterm=false
-if $cygwin; then
+if ${cygwin}; then
     case "$TERM" in
         rxvt* | xterm* | vt*) cygterm=true ;;
     esac
 fi
 
-if $cygterm; then
+if ${cygterm}; then
     JVM_OPTS="$JVM_OPTS -Djline.terminal=jline.UnixTerminal"
     stty -icanon min 1 -echo > /dev/null 2>&1
 fi
@@ -153,18 +159,18 @@ if [ "$1" = "self-install" ]; then
     echo "Downloading Dockstore to $DOCKSTORE_JAR now..."
     mkdir -p "$(dirname "$DOCKSTORE_JAR")"
 
-    if [ $SNAPSHOT = "YES" ]; then
-        DOCKSTORE_URL="file:///$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION.jar"
-    elif [ -r "/$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION.jar" ]; then
+    if [ ${SNAPSHOT} = "YES" ]; then
+        DOCKSTORE_URL="file:///$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION-shaded.jar"
+    elif [ -r "/$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION-shaded.jar" ]; then
     # for testing, if you have a local release version, just use it
-        DOCKSTORE_URL="file:///$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION.jar"
+        DOCKSTORE_URL="file:///$HOME/.m2/repository/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION-shaded.jar"
     else
-       DOCKSTORE_URL="https://seqwaremaven.oicr.on.ca/artifactory/collab-release/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION.jar"
+       DOCKSTORE_URL="https://artifacts.oicr.on.ca/artifactory/collab-release/io/dockstore/dockstore-client/$DOCKSTORE_VERSION/dockstore-client-$DOCKSTORE_VERSION-shaded.jar"
     fi
 
     echo "$DOCKSTORE_URL"
 
-    $HTTP_CLIENT "$DOCKSTORE_JAR.pending" "$DOCKSTORE_URL"
+    ${HTTP_CLIENT} "$DOCKSTORE_JAR.pending" "$DOCKSTORE_URL"
     if [ $? == 0 ]; then
         # TODO: checksum
         mv -f "$DOCKSTORE_JAR.pending" "$DOCKSTORE_JAR"
@@ -193,24 +199,25 @@ elif [ "$1" = "init" ]; then
 
     if [ "$HASACCOUNT" = "Y" ] || [ "$HASACCOUNT" = "y" ]; then
         mkdir -p "$(dirname "$DOCKSTORE_SETTINGS")"
-        touch $DOCKSTORE_SETTINGS   
-        chmod 600 $DOCKSTORE_SETTINGS
+        touch ${DOCKSTORE_SETTINGS}
+        chmod 600 ${DOCKSTORE_SETTINGS}
 
         echo
 
         echo "What is your token?"
         read TOKEN
-        echo "token: $TOKEN" >> $DOCKSTORE_SETTINGS
+        echo "token: $TOKEN" >> ${DOCKSTORE_SETTINGS}
 
         echo "What is your server-url?"
         read SERVER_URL
-        echo "server-url: $SERVER_URL" >> $DOCKSTORE_SETTINGS
+        echo "server-url: $SERVER_URL" >> ${DOCKSTORE_SETTINGS}
 
         echo    
         echo "Created Dockstore settings file at $DOCKSTORE_SETTINGS"
         echo
+        echo "Downloading default file-provisioning plugins"
+        ${JAVA_CMD} ${DOCKSTORE_JVM_OPTS} io.dockstore.client.cli.Client plugin download
         echo "Dockstore is ready to use!"
-
     else
         echo "Please setup an account at http://www.dockstore.org/"
     fi
@@ -231,11 +238,11 @@ else
       fi
     fi
 
-    $JAVA_CMD $DOCKSTORE_JVM_OPTS io.dockstore.client.cli.Client "$@"
+    ${JAVA_CMD} ${DOCKSTORE_JVM_OPTS} io.dockstore.client.cli.Client "$@"
 
     EXIT_CODE=$?
 
-    if $cygterm ; then
+    if ${cygterm} ; then
         stty icanon echo > /dev/null 2>&1
     fi
 

diff --git a/DockstoreRunner.py b/DockstoreRunner.py
@@ -29,6 +29,7 @@
 
 import logging
 import hashlib
+import errno
 from functools import partial
 
 import os

diff --git a/README.md b/README.md
@@ -99,15 +99,15 @@ Most people, other than developers of this tool, will use the Dockstore CLI to i
 
     # patch in /usr/local/lib/python2.7/dist-packages/cwltool
     # make a tmpdir like /datastore
-    docker build -t quay.io/ucsc_cgl/dockstore-tool-runner:1.0.12 .
+    docker build -t quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13 .
     # fill in your JSON from Dockstore.json template as Dockstore.my.json
     mkdir /datastore; chown ubuntu:ubuntu /datastore/
     # local execution
     TMPDIR=/datastore dockstore tool launch --entry Dockstore.cwl --local-entry --json Dockstore.my.json
     # as root in /datastore
     TMPDIR=/datastore dockstore tool launch --entry ~ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.cwl --local-entry --json ~ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.my.json
     # execute published on dockstore (this is the way most people will use this tool!)
-    dockstore tool launch --entry quay.io/ucsc_cgl/dockstore-tool-runner:1.0.12 --json Dockstore.my.json
+    dockstore tool launch --entry quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13 --json Dockstore.my.json
 
     # running you see it launch the cwltool command, you mind find this useful while debugging
     cwltool --enable-dev --non-strict --enable-net --outdir /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/outputs/ --tmpdir-prefix /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/working/ /home/ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.cwl /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/workflow_params.json

diff --git a/job.patch b/job.patch
@@ -1,6 +1,6 @@
---- job.py	2016-11-22 02:16:21.207420139 +0000
-+++ job_new.py	2016-11-22 02:16:21.207420139 +0000
-@@ -96,6 +96,10 @@
+--- job.py	2017-05-11 14:49:28.000000000 -0700
++++ job_new.py	2017-05-11 14:41:49.000000000 -0700
+@@ -147,6 +147,10 @@
 
          if img_id:
              runtime = ["docker", "run", "-i"]
@@ -11,20 +11,20 @@
              for src in self.pathmapper.files():
                  vol = self.pathmapper.mapper(src)
                  if vol.type == "File":
-@@ -105,9 +109,9 @@
+@@ -156,9 +160,9 @@
                      with open(createtmp, "w") as f:
                          f.write(vol.resolved.encode("utf-8"))
                      runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target))
--            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl"))
--            runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp"))
--            runtime.append(u"--workdir=%s" % ("/var/spool/cwl"))
+-            runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.outdir), self.builder.outdir))
+-            runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.tmpdir), "/tmp"))
+-            runtime.append(u"--workdir=%s" % (self.builder.outdir))
 +            #runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl"))
 +            runtime.append(u"--volume=%s:%s:rw" % (os.path.normpath(os.path.dirname(self.tmpdir)), "/tmp"))
 +            runtime.append(u"--workdir=%s" % (os.path.normpath(os.path.dirname(self.outdir))))
              runtime.append("--read-only=true")
-             if (kwargs.get("enable_net", None) is None and
-                     kwargs.get("custom_net", None) is not None):
-@@ -124,12 +128,12 @@
+
+             if kwargs.get("custom_net", None) is not None:
+@@ -177,12 +181,12 @@
              if rm_container:
                  runtime.append("--rm")
 
@@ -34,34 +34,41 @@
              # spec currently says "HOME must be set to the designated output
              # directory." but spec might change to designated temp directory.
              # runtime.append("--env=HOME=/tmp")
--            runtime.append("--env=HOME=/var/spool/cwl")
+-            runtime.append("--env=HOME=%s" % self.builder.outdir)
 +            runtime.append("--env=HOME="+os.path.normpath(os.path.dirname(self.outdir)))
 
-             for t,v in self.environment.items():
+             for t, v in self.environment.items():
                  runtime.append(u"--env=%s=%s" % (t, v))
-@@ -137,9 +141,9 @@
+@@ -190,8 +194,8 @@
              runtime.append(img_id)
          else:
              env = self.environment
 -            if not os.path.exists(self.tmpdir):
 -                os.makedirs(self.tmpdir)
--            env["TMPDIR"] = self.tmpdir
 +            if not os.path.exists(os.path.normpath(os.path.dirname(self.tmpdir))):
 +                os.makedirs(os.path.normpath(os.path.dirname(self.tmpdir)))
-+            env["TMPDIR"] = os.path.normpath(os.path.dirname(self.tmpdir))
              vars_to_preserve = kwargs.get("preserve_environment")
-             if vars_to_preserve is not None:
-                 for key, value in os.environ.items():
-@@ -161,7 +165,7 @@
+             if kwargs.get("preserve_entire_environment"):
+                 vars_to_preserve = os.environ
+@@ -200,7 +204,7 @@
+                     if key in vars_to_preserve and key not in env:
+                         env[key] = value
+             env["HOME"] = self.outdir
+-            env["TMPDIR"] = self.tmpdir
++            env["TMPDIR"] = os.path.normpath(os.path.dirname(self.tmpdir))
+
+             stageFiles(self.pathmapper, os.symlink)
+
+@@ -214,7 +218,7 @@
 
          _logger.info(u"[job %s] %s$ %s%s%s%s",
                       self.name,
 -                     self.outdir,
 +                     os.path.normpath(os.path.dirname(self.outdir)),
-                      " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
+                      " \\\n    ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in
+                                        (runtime + self.command_line)]),
                       u' < %s' % self.stdin if self.stdin else '',
-                      u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '',
-@@ -174,8 +178,8 @@
+@@ -228,8 +232,8 @@
 
          try:
              if self.generatefiles["listing"]:
@@ -71,44 +78,35 @@
 +                                            os.path.normpath(os.path.dirname(self.outdir)), separateDirs=False)
                  _logger.debug(u"[job %s] initial work dir %s", self.name,
                                json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4))
-                 def linkoutdir(src, tgt):
-@@ -193,7 +197,7 @@
-                 stdin = subprocess.PIPE
 
+@@ -249,7 +253,7 @@
+
+             stderr_path = None
              if self.stderr:
 -                abserr = os.path.join(self.outdir, self.stderr)
 +                abserr = os.path.join(os.path.normpath(os.path.dirname(self.outdir)), self.stderr)
                  dnerr = os.path.dirname(abserr)
                  if dnerr and not os.path.exists(dnerr):
                      os.makedirs(dnerr)
-@@ -202,7 +206,7 @@
-                 stderr = sys.stderr
+@@ -257,7 +261,7 @@
 
+             stdout_path = None
              if self.stdout:
 -                absout = os.path.join(self.outdir, self.stdout)
 +                absout = os.path.join(os.path.normpath(os.path.dirname(self.outdir)), self.stdout)
                  dn = os.path.dirname(absout)
                  if dn and not os.path.exists(dn):
                      os.makedirs(dn)
-@@ -217,7 +221,7 @@
-                                   stderr=stderr,
-                                   stdout=stdout,
-                                   env=env,
--                                  cwd=self.outdir)
-+                                  cwd=os.path.normpath(os.path.dirname(self.outdir)))
-
-             if sp.stdin:
-                 sp.stdin.close()
-@@ -253,7 +257,7 @@
-                         os.symlink(src, tgt)
-                 stageFiles(generatemapper, linkoutdir)
+@@ -295,7 +299,7 @@
+
+                 stageFiles(generatemapper, linkoutdir, ignoreWritable=True)
 
 -            outputs = self.collect_outputs(self.outdir)
 +            outputs = self.collect_outputs(os.path.normpath(os.path.dirname(self.outdir)))
 
          except OSError as e:
              if e.errno == 2:
-@@ -284,9 +288,9 @@
+@@ -328,12 +332,12 @@
              shutil.rmtree(self.stagedir, True)
 
          if rm_tmpdir:
@@ -122,3 +120,15 @@
 +        if move_outputs == "move" and empty_subtree(os.path.normpath(os.path.dirname(self.outdir))):
 +            _logger.debug(u"[job %s] Removing empty output directory %s", self.name, os.path.normpath(os.path.dirname(self.outdir)))
 +            shutil.rmtree(os.path.normpath(os.path.dirname(self.outdir)), True)
+
+
+ def _job_popen(
+@@ -380,7 +384,7 @@
+                               stdout=stdout,
+                               stderr=stderr,
+                               env=env,
+-                              cwd=cwd)
++                              cwd=os.path.normpath(os.path.dirname(self.outdir)))
+
+         if sp.stdin:
+             sp.stdin.close()