Merge branch 'release/1.0.14'

Dockerfile

@@ -1,13 +1,9 @@
-FROM quay.io/ucsc_cgl/redwood-client:1.1.1
+FROM quay.io/ucsc_cgl/redwood-client:1.2.0
 
 MAINTAINER Walt Shands jshands@ucsc.edu
 
-#patch logback.xml so the storage client writes its log files to /tmp because other
-#directories are readonly when cwltool runs the container
-RUN sed -i 's~<property name=\"log.dir\" value=\"${logging.path:-/tmp/icgc-storage-client/logs}\" />~<property name=\"log.dir\" value=\"/tmp/icgc-storage-client/logs\" />~g' /dcc/icgc-storage-client/conf/logback.xml
-
-RUN sed -i 's~<property name=\"log.dir\" value=\"${LOG_PATH:-../logs}\" />~<property name=\"log.dir\" value=\"/tmp/dcc-metadata-client/logs\" />~g' /dcc/dcc-metadata-client/conf/logback.xml
-
+#Set the log directory for redwood to be under /tmp since the rest of the file system is locked
+ENV REDWOOD_LOG_DIR /tmp
 
 WORKDIR ./
 
@@ -52,6 +48,8 @@ COPY job.patch /usr/local/lib/python2.7/dist-packages/cwltool/job.patch
 RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/job.patch
 COPY process.patch /usr/local/lib/python2.7/dist-packages/cwltool/process.patch
 RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/process.patch
+COPY main.patch /usr/local/lib/python2.7/dist-packages/cwltool/main.patch
+RUN patch -d /usr/local/lib/python2.7/dist-packages/cwltool/ < /usr/local/lib/python2.7/dist-packages/cwltool/main.patch
 
 #Add ubuntu user and group
 RUN groupadd -r -g 1000 ubuntu && useradd -r -g ubuntu -u 1000 ubuntu
@@ -72,6 +70,7 @@ ENV PATH /home/ubuntu/Dockstore/:$PATH
 #copy dockstore files to root so root can run dockstore
 COPY .dockstore/ /root/.dockstore
 COPY Dockstore/ /root/Dockstore
+RUN mkdir /root/.dockstore/plugins
 RUN chmod a+x /root/Dockstore/dockstore
 
 ENV PATH /root/Dockstore/:$PATH

Dockstore.cwl

@@ -24,7 +24,7 @@ dct:creator:
 
 requirements:
   - class: DockerRequirement
-    dockerPull: "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13"
+    dockerPull: "quay.io/ucsc_cgl/dockstore-tool-runner:1.0.14"
 hints:
   - class: ResourceRequirement
     coresMin: 1

README.md

@@ -99,15 +99,15 @@ Most people, other than developers of this tool, will use the Dockstore CLI to i
 
     # patch in /usr/local/lib/python2.7/dist-packages/cwltool
     # make a tmpdir like /datastore
-    docker build -t quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13 .
+    docker build -t quay.io/ucsc_cgl/dockstore-tool-runner:1.0.14 .
     # fill in your JSON from Dockstore.json template as Dockstore.my.json
     mkdir /datastore; chown ubuntu:ubuntu /datastore/
     # local execution
     TMPDIR=/datastore dockstore tool launch --entry Dockstore.cwl --local-entry --json Dockstore.my.json
     # as root in /datastore
     TMPDIR=/datastore dockstore tool launch --entry ~ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.cwl --local-entry --json ~ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.my.json
     # execute published on dockstore (this is the way most people will use this tool!)
-    dockstore tool launch --entry quay.io/ucsc_cgl/dockstore-tool-runner:1.0.13 --json Dockstore.my.json
+    dockstore tool launch --entry quay.io/ucsc_cgl/dockstore-tool-runner:1.0.14 --json Dockstore.my.json
 
     # running you see it launch the cwltool command, you mind find this useful while debugging
     cwltool --enable-dev --non-strict --enable-net --outdir /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/outputs/ --tmpdir-prefix /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/working/ /home/ubuntu/gitroot/BD2KGenomics/dcc-dockstore-tool-runner/Dockstore.cwl /datastore/./datastore/launcher-ff6b55b3-52e8-430c-9a70-1ff295332698/workflow_params.json

job.patch

@@ -1,6 +1,14 @@
---- job.py	2017-05-11 14:49:28.000000000 -0700
-+++ job_new.py	2017-05-11 14:41:49.000000000 -0700
-@@ -147,6 +147,10 @@
+--- job.py	2017-05-18 20:30:01.573793271 +0000
++++ job_new.py	2017-05-22 21:16:35.464979861 +0000
+@@ -34,7 +34,6 @@
+ import json
+ import sys
+ import subprocess
+-
+ with open(sys.argv[1], "r") as f:
+     popen_description = json.load(f)
+     commands = popen_description["commands"]
+@@ -147,6 +146,10 @@
 
          if img_id:
              runtime = ["docker", "run", "-i"]
@@ -11,7 +19,7 @@
              for src in self.pathmapper.files():
                  vol = self.pathmapper.mapper(src)
                  if vol.type == "File":
-@@ -156,9 +160,9 @@
+@@ -156,9 +159,9 @@
                      with open(createtmp, "w") as f:
                          f.write(vol.resolved.encode("utf-8"))
                      runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target))
@@ -24,7 +32,7 @@
              runtime.append("--read-only=true")
 
              if kwargs.get("custom_net", None) is not None:
-@@ -177,12 +181,12 @@
+@@ -177,12 +180,12 @@
              if rm_container:
                  runtime.append("--rm")
 
@@ -39,7 +47,7 @@
 
              for t, v in self.environment.items():
                  runtime.append(u"--env=%s=%s" % (t, v))
-@@ -190,8 +194,8 @@
+@@ -190,8 +193,8 @@
              runtime.append(img_id)
          else:
              env = self.environment
@@ -50,7 +58,7 @@
              vars_to_preserve = kwargs.get("preserve_environment")
              if kwargs.get("preserve_entire_environment"):
                  vars_to_preserve = os.environ
-@@ -200,7 +204,7 @@
+@@ -200,7 +203,7 @@
                      if key in vars_to_preserve and key not in env:
                          env[key] = value
              env["HOME"] = self.outdir
@@ -59,7 +67,7 @@
 
              stageFiles(self.pathmapper, os.symlink)
 
-@@ -214,7 +218,7 @@
+@@ -214,7 +217,7 @@
 
          _logger.info(u"[job %s] %s$ %s%s%s%s",
                       self.name,
@@ -68,7 +76,7 @@
                       " \\\n    ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in
                                         (runtime + self.command_line)]),
                       u' < %s' % self.stdin if self.stdin else '',
-@@ -228,8 +232,8 @@
+@@ -228,8 +231,8 @@
 
          try:
              if self.generatefiles["listing"]:
@@ -79,7 +87,7 @@
                  _logger.debug(u"[job %s] initial work dir %s", self.name,
                                json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4))
 
-@@ -249,7 +253,7 @@
+@@ -249,7 +252,7 @@
 
              stderr_path = None
              if self.stderr:
@@ -88,7 +96,7 @@
                  dnerr = os.path.dirname(abserr)
                  if dnerr and not os.path.exists(dnerr):
                      os.makedirs(dnerr)
-@@ -257,7 +261,7 @@
+@@ -257,7 +260,7 @@
 
              stdout_path = None
              if self.stdout:
@@ -97,7 +105,16 @@
                  dn = os.path.dirname(absout)
                  if dn and not os.path.exists(dn):
                      os.makedirs(dn)
-@@ -295,7 +299,7 @@
+@@ -270,7 +273,7 @@
+                 stdout_path=stdout_path,
+                 stderr_path=stderr_path,
+                 env=env,
+-                cwd=self.outdir,
++                cwd=os.path.normpath(os.path.dirname(self.outdir)),
+                 build_job_script=build_job_script,
+             )
+
+@@ -295,7 +298,7 @@
 
                  stageFiles(generatemapper, linkoutdir, ignoreWritable=True)
 
@@ -106,7 +123,7 @@
 
          except OSError as e:
              if e.errno == 2:
-@@ -328,12 +332,12 @@
+@@ -328,12 +331,12 @@
              shutil.rmtree(self.stagedir, True)
 
          if rm_tmpdir:
@@ -123,12 +140,3 @@
 
 
  def _job_popen(
-@@ -380,7 +384,7 @@
-                               stdout=stdout,
-                               stderr=stderr,
-                               env=env,
--                              cwd=cwd)
-+                              cwd=os.path.normpath(os.path.dirname(self.outdir)))
-
-         if sp.stdin:
-             sp.stdin.close()

job_new.py

@@ -34,7 +34,6 @@
 import json
 import sys
 import subprocess
-
 with open(sys.argv[1], "r") as f:
     popen_description = json.load(f)
     commands = popen_description["commands"]
@@ -274,7 +273,7 @@ def linkoutdir(src, tgt):
                 stdout_path=stdout_path,
                 stderr_path=stderr_path,
                 env=env,
-                cwd=self.outdir,
+                cwd=os.path.normpath(os.path.dirname(self.outdir)),
                 build_job_script=build_job_script,
             )
 
@@ -384,7 +383,7 @@ def _job_popen(
                               stdout=stdout,
                               stderr=stderr,
                               env=env,
-                              cwd=os.path.normpath(os.path.dirname(self.outdir)))
+                              cwd=cwd)
 
         if sp.stdin:
             sp.stdin.close()

main.patch

@@ -0,0 +1,21 @@
+--- main.py	2017-05-23 18:51:12.839227841 +0000
++++ main_new.py	2017-05-23 22:41:39.979009776 +0000
+@@ -200,7 +200,7 @@
+     output_dirs = set()
+     finaloutdir = kwargs.get("outdir")
+     kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs["tmp_outdir_prefix"]) if kwargs.get(
+-        "tmp_outdir_prefix") else tempfile.mkdtemp()
++        "tmp_outdir_prefix") else tempfile.mkdtemp() 
+     output_dirs.add(kwargs["outdir"])
+
+     jobReqs = None
+@@ -612,7 +612,8 @@
+                      'validate': False}.iteritems():
+             if not hasattr(args, k):
+                 setattr(args, k, v)
+-
++        #HACK; Setting tmp_outdir_prefix to /datastore/
++        setattr(args, 'tmp_outdir_prefix', '/datastore/') 
+         if args.quiet:
+             _logger.setLevel(logging.WARN)
+         if args.debug: