Allow describing wdl with zipped imports.

Author: kshakir

CHANGELOG.md

@@ -7,6 +7,14 @@
 Cromwell now allows opting into configured soft links on shared file systems such as HPC environments. More details can
 be found [here](https://cromwell.readthedocs.io/en/stable/backends/HPC/#optional-docker-soft-links).
 
+### `/describe` endpoint support for `workflowDependencies`
+
+Previously it was not possible to describe a `workflowSource` that required `workflowDependencies` as the `/describe`
+endpoint did not allow specifying the additional zip file.
+
+Now the endpoint will allow the user to upload the `workflowDependencies` zip file, will validate the top level
+workflow plus the dependencies, then return the appropriate response including the defined workflow inputs and outputs.
+
 ## 87 Release Notes
 
 ### `upgrade` command removed from Womtool

CromIAM/src/main/resources/swagger/cromiam.yaml

@@ -838,6 +838,11 @@ paths:
           required: false
           type: file
           in: formData
+        - name: workflowDependencies
+          description: ZIP file containing workflow source files that are used to resolve local imports. This zip bundle will be unpacked in a sandbox accessible to this workflow.
+          required: false
+          type: file
+          in: formData
         - $ref: '#/parameters/workflowTypeParam'
         - $ref: '#/parameters/workflowTypeVersionParam'
       responses:

centaur/src/main/scala/centaur/test/Test.scala

@@ -244,8 +244,7 @@ object Operations extends StrictLogging {
       }
 
       override def run: IO[Unit] =
-        // We can't describe workflows based on zipped imports, so don't try:
-        if (workflow.skipDescribeEndpointValidation || workflow.data.zippedImports.nonEmpty) {
+        if (workflow.skipDescribeEndpointValidation) {
           IO.pure(())
         } else {
           checkDescriptionInner(0)

centaur/src/main/scala/centaur/test/workflow/Workflow.scala

@@ -45,7 +45,8 @@ final case class Workflow private (testName: String,
     workflowUrl = data.workflowUrl,
     workflowType = data.workflowType,
     workflowTypeVersion = data.workflowTypeVersion,
-    inputsJson = data.inputs.map(_.unsafeRunSync())
+    inputsJson = data.inputs.map(_.unsafeRunSync()),
+    zippedImports = data.zippedImports
   )
 
   def secondRun: Workflow =

cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala

@@ -266,7 +266,13 @@ object CromwellClient {
       Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source)))
     }
 
-    val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq: _*)
+    val zipBodyParts = Map(
+      "workflowDependencies" -> describeRequest.zippedImports
+    ) collect { case (name, Some(file)) =>
+      Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path)
+    }
+
+    val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq: _*)
     multipartFormData.toEntity()
   }
 

cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala

@@ -1,8 +1,11 @@
 package cromwell.api.model
 
+import better.files.File
+
 final case class WorkflowDescribeRequest(workflowSource: Option[String],
                                          workflowUrl: Option[String],
                                          workflowType: Option[String],
                                          workflowTypeVersion: Option[String],
-                                         inputsJson: Option[String]
+                                         inputsJson: Option[String],
+                                         zippedImports: Option[File]
 )

docs/api/RESTAPI.md

@@ -662,6 +662,11 @@ paths:
           required: false
           type: file
           in: formData
+        - name: workflowDependencies
+          description: ZIP file containing workflow source files that are used to resolve local imports. This zip bundle will be unpacked in a sandbox accessible to this workflow.
+          required: false
+          type: file
+          in: formData
         - $ref: '#/parameters/workflowTypeParam'
         - $ref: '#/parameters/workflowTypeVersionParam'
       responses:

engine/src/main/resources/swagger/cromwell.yaml

@@ -60,6 +60,7 @@ trait WomtoolRouteSupport extends WebServiceUtils with GithubAuthVendingSupport
     val workflowInputs = data.get("workflowInputs").map(_.utf8String)
     val workflowType = data.get("workflowType").map(_.utf8String)
     val workflowVersion = data.get("workflowTypeVersion").map(_.utf8String)
+    val workflowDependencies = data.get("workflowDependencies").map(_.toArray)
 
     val wsfc = WorkflowSourceFilesCollection(
       workflowSource,
@@ -70,7 +71,7 @@ trait WomtoolRouteSupport extends WebServiceUtils with GithubAuthVendingSupport
       workflowInputs.getOrElse(""),
       workflowOptions = WorkflowOptions.empty,
       labelsJson = "",
-      importsFile = None,
+      importsFile = workflowDependencies,
       workflowOnHold = false,
       warnings = Seq.empty,
       requestedWorkflowId = None

engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala

@@ -761,7 +761,9 @@ object CromwellApiServiceSpec {
               s"[reading back DescribeRequest contents] workflow url: ${sourceFiles.workflowUrl}",
               s"[reading back DescribeRequest contents] inputs: ${sourceFiles.inputsJson}",
               s"[reading back DescribeRequest contents] type: ${sourceFiles.workflowType}",
-              s"[reading back DescribeRequest contents] version: ${sourceFiles.workflowTypeVersion}"
+              s"[reading back DescribeRequest contents] version: ${sourceFiles.workflowTypeVersion}",
+              s"[reading back DescribeRequest contents] dependencies: ${sourceFiles.importsZipFileOption
+                  .map(bytes => bytes.map(b => "0x%02X".format(b)).mkString("[", ", ", "]"))}"
             )
 
             sender() ! DescribeSuccess(description =