Merge pull request #17 from dnanexus/cwlpack

APPS-897 Switch from cwltool to sbpack for packing CWL files

.gitignore

@@ -4,9 +4,10 @@ src/main/java
 src/main/antlr4/*.tokens
 src/main/antlr4/gen/
 gen/
+.bsp/
 
 # VSCcode
 .metals/
 .vscode/
 
-.java-version
+.java-version

RELEASE_NOTES.md

@@ -1,5 +1,22 @@
 # Change log
 
+## in develop
+
+* **Breaking**
+  * Parser API has changed substantially, with parameters added, removed, and rearranged
+  * `Parser.parse` method is now private - use `parseString` or `parseFile` instead
+  * Removes all options to modify IDs during parsing
+* Handles workflows packed by cwlpack
+* Adds `Process.simpleName` method to return simplified process name from ID automatically generated by `cwlpack --add-ids`
+* Adds `Identifiable.copySimplifyIds` method to deep-copy objects with simplified IDs
+* Adds `CwlEnum.symbolNames` function for getting enum symbols without any namespace prefixes
+* Adds `coerce` option to `Evaluator.evaluate` to actually perform type coercion, rather than just checking that the result is coercible to the specified type
+* Trims `StringValue` when coercing to primitive types
+* Fixes `Evaluator.finalizeInputValue` for compound and optional types
+* `Evaluator.finalizeInputValue` loads file contents from remote file source if file does not exist locally
+* `CwlType.coerceTo` now returns both the coerced-to type and value
+* Added `CwlType.CwlGenericRecord`, which is coercible to either `CwlInputRecord` or `CwlOutputRecord`
+
 ## 0.7.4 (2021-01-05)
 
 * Updates code to compile with JDK11

build.sbt

@@ -40,44 +40,51 @@ lazy val root = project
       version := getVersion,
       settings,
       assemblySettings,
-      libraryDependencies ++= Seq(
-          dependencies.dxCommon,
-          dependencies.dxYaml,
-          dependencies.typesafe,
-          dependencies.spray,
-          dependencies.rhino,
-          dependencies.antlr,
-          dependencies.scalatest,
-          dependencies.snakeyaml,
-          dependencies.junit
-      ),
+      libraryDependencies ++= dependencies,
       assembly / assemblyJarName := "cwlScala.jar"
   )
 
-lazy val dependencies = new {
+lazy val dependencies = {
   val dxCommonVersion = "0.11.0"
   val dxYamlVersion = "0.1.1"
   val typesafeVersion = "1.4.1"
   val sprayVersion = "1.3.6"
   val scalatestVersion = "3.2.9"
   val yamlVersion = "2.3"
+  val jacksonDatabindVersion = "2.13.1"
+  val jacksonDataformatYamlVersion = "2.13.1"
   val rhinoVersion = "1.7.13"
   val antlr4Version = "4.9.3"
-  val junitVersion = "4.13.2"
+  //val junitVersion = "4.13.2"
 
   val dxCommon = "com.dnanexus" % "dxcommon" % dxCommonVersion
   val dxYaml = "com.dnanexus" % "dxyaml" % dxYamlVersion
   val typesafe = "com.typesafe" % "config" % typesafeVersion
   val spray = "io.spray" %% "spray-json" % sprayVersion
   // cwljava dependencies
   val snakeyaml = "org.snakeyaml" % "snakeyaml-engine" % yamlVersion
+  val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion
+  val jacksonDataformatYaml = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonDataformatYamlVersion
   // rhino dependencies
   val rhino = "org.mozilla" % "rhino" % rhinoVersion
   // antlr4 dependencies
   val antlr = "org.antlr" % "antlr4" % antlr4Version
   //---------- Test libraries -------------------//
-  val junit = "junit" % "junit" % junitVersion % Test
+  //val junit = "junit" % "junit" % junitVersion % Test
   val scalatest = "org.scalatest" % "scalatest_2.13" % scalatestVersion % Test
+
+  Seq(
+      dxCommon,
+      dxYaml,
+      typesafe,
+      spray,
+      rhino,
+      antlr,
+      scalatest,
+      snakeyaml,
+      jacksonDatabind,
+      jacksonDataformatYaml
+  )
 }
 
 val githubDxScalaResolver = Resolver.githubPackages("dnanexus", "dxScala")

src/main/scala/dx/cwl/CommandLineTool.scala

@@ -30,7 +30,7 @@ case class CommandInputBinding(position: CwlValue,
                                valueFrom: Option[CwlValue])
 
 object CommandInputBinding {
-  def apply(binding: CommandLineBindingImpl,
+  def parse(binding: CommandLineBindingImpl,
             schemaDefs: Map[String, CwlSchema]): CommandInputBinding = {
     CommandInputBinding(
         translateOptionalObject(binding.getPosition)
@@ -58,24 +58,40 @@ case class CommandInputParameter(id: Option[Identifier],
                                  loadContents: Boolean,
                                  loadListing: LoadListing.LoadListing)
     extends InputParameter
-    with Loadable
+    with Loadable {
+  override def copySimplifyIds(dropNamespace: Boolean,
+                               replacePrefix: (Either[Boolean, String], Option[String]),
+                               simplifyAutoNames: Boolean,
+                               dropCwlExtension: Boolean): CommandInputParameter = {
+    copy(
+        id = id.map(
+            _.simplify(dropNamespace, replacePrefix, simplifyAutoNames, dropCwlExtension)
+        ),
+        cwlType = CwlType.copySimplifyIds(cwlType,
+                                          dropNamespace,
+                                          replacePrefix,
+                                          simplifyAutoNames,
+                                          dropCwlExtension)
+    )
+  }
+}
 
 object CommandInputParameter {
-  def apply(
+  def parse(
       param: CommandInputParameterImpl,
       schemaDefs: Map[String, CwlSchema],
-      stripFragPrefix: Option[String] = None,
       defaultNamespace: Option[String] = None
   ): (CommandInputParameter, Boolean) = {
     val (types, stdfile) = CwlType.translate(param.getType, schemaDefs)
     val inparam = CommandInputParameter(
-        translateOptional(param.getId).map(Identifier.parse(_, stripFragPrefix, defaultNamespace)),
+        translateOptional(param.getId)
+          .map(Identifier.parse(_, defaultNamespace)),
         translateOptional(param.getLabel),
         translateDoc(param.getDoc),
         types,
         translateOptional(param.getDefault).map(CwlValue(_, schemaDefs)),
         translateOptional(param.getInputBinding).map {
-          case binding: CommandLineBindingImpl => CommandInputBinding(binding, schemaDefs)
+          case binding: CommandLineBindingImpl => CommandInputBinding.parse(binding, schemaDefs)
           case other =>
             throw new RuntimeException(s"unexpected CommandLineBinding value ${other}")
         },
@@ -97,7 +113,7 @@ case class CommandOutputBinding(glob: Vector[CwlValue],
     extends Loadable
 
 object CommandOutputBinding {
-  def apply(binding: CommandOutputBindingImpl,
+  def parse(binding: CommandOutputBindingImpl,
             schemaDefs: Map[String, CwlSchema]): CommandOutputBinding = {
     CommandOutputBinding(
         translateOptionalArray(binding.getGlob).map(CwlValue(_, schemaDefs)),
@@ -117,21 +133,36 @@ case class CommandOutputParameter(id: Option[Identifier],
                                   secondaryFiles: Vector[SecondaryFile],
                                   format: Option[CwlValue],
                                   streamable: Boolean)
-    extends OutputParameter
+    extends OutputParameter {
+  override def copySimplifyIds(dropNamespace: Boolean,
+                               replacePrefix: (Either[Boolean, String], Option[String]),
+                               simplifyAutoNames: Boolean,
+                               dropCwlExtension: Boolean): CommandOutputParameter = {
+    copy(
+        id = id.map(
+            _.simplify(dropNamespace, replacePrefix, simplifyAutoNames, dropCwlExtension)
+        ),
+        cwlType = CwlType.copySimplifyIds(cwlType,
+                                          dropNamespace,
+                                          replacePrefix,
+                                          simplifyAutoNames,
+                                          dropCwlExtension)
+    )
+  }
+}
 
 object CommandOutputParameter {
-  def apply(
+  def parse(
       param: CommandOutputParameterImpl,
       schemaDefs: Map[String, CwlSchema],
-      stripFragPrefix: Option[String] = None,
       defaultNamespace: Option[String] = None
   ): (CommandOutputParameter, Option[StdFile.StdFile]) = {
     val (types, stdfile) = CwlType.translate(param.getType, schemaDefs)
     val outputBinding = translateOptional(param.getOutputBinding) match {
       case Some(_) if stdfile.nonEmpty =>
         throw new RuntimeException(s"outputBinding not allowed for type ${stdfile.get}")
       case Some(binding: CommandOutputBindingImpl) =>
-        Some(CommandOutputBinding(binding, schemaDefs))
+        Some(CommandOutputBinding.parse(binding, schemaDefs))
       case None if stdfile.nonEmpty =>
         Some(
             CommandOutputBinding(glob = Vector(RandomFile(stdfile.get)),
@@ -149,7 +180,8 @@ object CommandOutputParameter {
       translateOptional(param.getStreamable).exists(_.booleanValue())
     }
     val outparam = CommandOutputParameter(
-        translateOptional(param.getId).map(Identifier.parse(_, stripFragPrefix, defaultNamespace)),
+        translateOptional(param.getId)
+          .map(Identifier.parse(_, defaultNamespace)),
         translateOptional(param.getLabel),
         translateDoc(param.getDoc),
         types,
@@ -189,42 +221,60 @@ case class CommandLineTool(source: Option[String],
                            successCodes: Set[Int],
                            temporaryFailCodes: Set[Int],
                            permanentFailCodes: Set[Int])
-    extends Process
+    extends Process {
+  override def copySimplifyIds(
+      dropNamespace: Boolean,
+      replacePrefix: (Either[Boolean, String], Option[String]),
+      simplifyAutoNames: Boolean,
+      dropCwlExtension: Boolean
+  ): CommandLineTool = {
+    val (simplifiedId, childReplacePrefix) =
+      getIdAndChildReplacePrefix(dropNamespace, replacePrefix, simplifyAutoNames, dropCwlExtension)
+    copy(
+        id = simplifiedId,
+        inputs = inputs.map(
+            _.copySimplifyIds(dropNamespace,
+                              childReplacePrefix,
+                              simplifyAutoNames,
+                              dropCwlExtension)
+        ),
+        outputs = outputs.map(
+            _.copySimplifyIds(dropNamespace,
+                              childReplacePrefix,
+                              simplifyAutoNames,
+                              dropCwlExtension)
+        )
+    )
+  }
+}
 
 object CommandLineTool {
 
   /**
     * Creates a [[CommandLineTool]] from the [[CommandLineToolImpl]] created by the Java parser.
-    * @param tool the Java object
-    * @param ctx the Parser
-    * @param source the CWL original source code
-    * @param defaultFrag an optional tool name, to use if it is not specified in the document
+    * @param tool the Java object.
+    * @param ctx the Parser.
+    * @param source the CWL original source code.
+    * @param defaultNamespace the namespace to use when creating Identifiers when one is not present.
+    * @param defaultMainFrag an optional tool name, to use if it is not specified in the document.
     * @return a [[CommandLineTool]]
     */
-  def apply(tool: CommandLineToolImpl,
+  def parse(tool: CommandLineToolImpl,
             ctx: Parser,
             source: Option[Path] = None,
             defaultNamespace: Option[String],
-            defaultFrag: Option[String] = None,
-            isGraph: Boolean = false): CommandLineTool = {
+            defaultMainFrag: Option[String] = None): CommandLineTool = {
+    val rawId = Identifier.get(tool.getId, defaultNamespace, defaultMainFrag, source)
+    val toolId = rawId.orElse {
+      val namespace = rawId.map(_.namespace).getOrElse(defaultNamespace)
+      defaultMainFrag
+        .map(frag => Identifier(namespace, frag))
+        .orElse(
+            Option.when(source.isDefined)(Identifier.fromSource(source.get, namespace))
+        )
+    }
     val (requirements, allSchemaDefs) =
       Requirement.applyRequirements(tool.getRequirements, ctx.schemaDefs)
-
-    val rawId = Identifier.get(tool.getId, defaultNamespace, defaultFrag, source)
-    val stripFragPrefix = if (isGraph) rawId.flatMap(_.frag.map(p => s"${p}/")) else None
-
-    val toolId = Option
-      .when(isGraph && rawId.flatMap(_.frag).contains(Identifier.Main)) {
-        val namespace = rawId.map(_.namespace).getOrElse(defaultNamespace)
-        Option
-          .when(defaultFrag.isDefined)(Identifier(namespace, defaultFrag))
-          .orElse(
-              Option.when(source.isDefined)(Identifier.fromSource(source.get, namespace))
-          )
-      }
-      .flatten
-      .orElse(rawId)
-
     // An input may have type `stdin`, which is a file that is created from the
     // standard input piped to the CommandLineTool. A maximum of one input parameter
     // can have the `stdin` type, and if there is one then the tool's `stdin`
@@ -233,7 +283,7 @@ object CommandLineTool {
     val (inputParams, isStdin) = tool.getInputs.asScala
       .map {
         case param: CommandInputParameterImpl =>
-          CommandInputParameter(param, allSchemaDefs, stripFragPrefix, defaultNamespace)
+          CommandInputParameter.parse(param, allSchemaDefs, defaultNamespace)
         case other =>
           throw new RuntimeException(s"unexpected CommandInputParameter value ${other}")
       }
@@ -256,22 +306,21 @@ object CommandLineTool {
     val stdin = paramStdin
       .map { param =>
         val paramId = param.id
-          .flatMap(_.frag)
           .getOrElse(
               throw new Exception(s"missing input parameter ${paramStdin}")
           )
+          .frag
         CwlValue(s"$${inputs.${paramId}.path}", allSchemaDefs)
       }
       .orElse(toolStdin)
-
     // An output parameter may have type `stdout` or `stderr`. There can be a maximum of
     // one output parameter with each type, and if there is one then the corresponding
     // CommandLineTool attribute cannot also be set.
     // https://www.commonwl.org/v1.2/CommandLineTool.html#stdout
     val (outputParams, stdfile) = tool.getOutputs.asScala
       .map {
         case param: CommandOutputParameterImpl =>
-          CommandOutputParameter(param, allSchemaDefs, stripFragPrefix, defaultNamespace)
+          CommandOutputParameter.parse(param, allSchemaDefs, defaultNamespace)
         case other =>
           throw new RuntimeException(s"unexpected CommandOutputParameter value ${other}")
       }
@@ -289,14 +338,12 @@ object CommandLineTool {
           Some(RandomFile(StdFile.Stderr))
         case other => other
       }
-
     val arguments = translateOptionalArray(tool.getArguments).map {
       case binding: CommandLineBindingImpl =>
-        BindingArgument(CommandInputBinding(binding, allSchemaDefs))
+        BindingArgument(CommandInputBinding.parse(binding, allSchemaDefs))
       case expr =>
         ExprArgument(CwlValue(expr, allSchemaDefs))
     }
-
     CommandLineTool(
         source.map(_.toString),
         translateOptional(tool.getCwlVersion),