Skip to content

Commit 1d1c815

Browse files
kshakirkshakir
committed
Allow describing wdl with zipped imports.
1 parent b1ec16a commit 1d1c815

File tree

18 files changed

+201
-22
lines changed

18 files changed

+201
-22
lines changed

‎CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,14 @@ For more information, see the [Cromwell 79 release notes](https://github.com/bro
6565

6666
* Cromwell now attempts to translate `disks` attributes [written for GCP](https://cromwell.readthedocs.io/en/stable/RuntimeAttributes/#disks) into valid `disk` attributes for TES. For information on supported conversions, refer to the [TES documentation](https://cromwell.readthedocs.io/en/stable/backends/TES/).
6767

68+
### `/describe` endpoint support for `workflowDependencies`
69+
70+
Previously it was not possible to describe a `workflowSource` that required `workflowDependencies` as the `/describe`
71+
endpoint did not allow specifying the additional zip file.
72+
73+
Now the endpoint will allow the user to upload the `workflowDependencies` zip file, will validate the top level
74+
workflow plus the dependencies, then return the appropriate response including the defined workflow inputs and outputs.
75+
6876
### Bug Fixes
6977

7078
* Reference disks are only mounted if configured in the workflow options.

‎CromIAM/src/main/resources/swagger/cromiam.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -838,6 +838,11 @@ paths:
838838
required: false
839839
type: file
840840
in: formData
841+
- name: workflowDependencies
842+
description: ZIP file containing workflow source files that are used to resolve local imports. This zip bundle will be unpacked in a sandbox accessible to this workflow.
843+
required: false
844+
type: file
845+
in: formData
841846
- $ref: '#/parameters/workflowTypeParam'
842847
- $ref: '#/parameters/workflowTypeVersionParam'
843848
responses:

‎centaur/src/main/scala/centaur/test/Test.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,7 @@ object Operations extends StrictLogging {
228228
override def run: IO[Unit] = {
229229

230230

231-
// We can't describe workflows based on zipped imports, so don't try:
232-
if (workflow.skipDescribeEndpointValidation || workflow.data.zippedImports.nonEmpty) {
231+
if (workflow.skipDescribeEndpointValidation) {
233232
IO.pure(())
234233
} else {
235234
checkDescriptionInner(0)

‎centaur/src/main/scala/centaur/test/workflow/Workflow.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ final case class Workflow private(testName: String,
4343
workflowUrl = data.workflowUrl,
4444
workflowType = data.workflowType,
4545
workflowTypeVersion = data.workflowTypeVersion,
46-
inputsJson = data.inputs.map(_.unsafeRunSync())
46+
inputsJson = data.inputs.map(_.unsafeRunSync()),
47+
zippedImports = data.zippedImports,
4748
)
4849

4950
def secondRun: Workflow = {

‎cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,13 @@ object CromwellClient {
261261
Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source)))
262262
}
263263

264-
val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq : _*)
264+
val zipBodyParts = Map(
265+
"workflowDependencies" -> describeRequest.zippedImports
266+
) collect {
267+
case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path)
268+
}
269+
270+
val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*)
265271
multipartFormData.toEntity()
266272
}
267273

‎cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
package cromwell.api.model
22

3+
import better.files.File
4+
35
final case class WorkflowDescribeRequest(workflowSource: Option[String],
46
workflowUrl: Option[String],
57
workflowType: Option[String],
68
workflowTypeVersion: Option[String],
7-
inputsJson: Option[String])
9+
inputsJson: Option[String],
10+
zippedImports: Option[File],
11+
)

‎docs/api/RESTAPI.md

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎engine/src/main/resources/swagger/cromwell.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,11 @@ paths:
642642
required: false
643643
type: file
644644
in: formData
645+
- name: workflowDependencies
646+
description: ZIP file containing workflow source files that are used to resolve local imports. This zip bundle will be unpacked in a sandbox accessible to this workflow.
647+
required: false
648+
type: file
649+
in: formData
645650
- $ref: '#/parameters/workflowTypeParam'
646651
- $ref: '#/parameters/workflowTypeVersionParam'
647652
responses:

‎engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ trait WomtoolRouteSupport extends WebServiceUtils {
4545
val workflowInputs = data.get("workflowInputs").map(_.utf8String)
4646
val workflowType = data.get("workflowType").map(_.utf8String)
4747
val workflowVersion = data.get("workflowTypeVersion").map(_.utf8String)
48+
val workflowDependencies = data.get("workflowDependencies").map(_.toArray)
4849

4950
val wsfc = WorkflowSourceFilesCollection(
5051
workflowSource,
@@ -55,7 +56,7 @@ trait WomtoolRouteSupport extends WebServiceUtils {
5556
workflowInputs.getOrElse(""),
5657
workflowOptions = WorkflowOptions.empty,
5758
labelsJson = "",
58-
importsFile = None,
59+
importsFile = workflowDependencies,
5960
workflowOnHold = false,
6061
warnings = Seq.empty,
6162
requestedWorkflowId = None

‎engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,12 @@ object CromwellApiServiceSpec {
670670
s"[reading back DescribeRequest contents] workflow url: ${sourceFiles.workflowUrl}",
671671
s"[reading back DescribeRequest contents] inputs: ${sourceFiles.inputsJson}",
672672
s"[reading back DescribeRequest contents] type: ${sourceFiles.workflowType}",
673-
s"[reading back DescribeRequest contents] version: ${sourceFiles.workflowTypeVersion}"
673+
s"[reading back DescribeRequest contents] version: ${sourceFiles.workflowTypeVersion}",
674+
s"[reading back DescribeRequest contents] dependencies: ${
675+
sourceFiles.importsZipFileOption.map(bytes =>
676+
bytes.map(b => "0x%02X".format(b)).mkString("[", ", ", "]")
677+
)
678+
}",
674679
)
675680

676681
sender() ! DescribeSuccess(description = WorkflowDescription(valid = true, errors = readBack, validWorkflow = true))

‎engine/src/test/scala/cromwell/webservice/routes/WomtoolRouteSupportSpec.scala

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with
3535
val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, "{\"a\":\"is for apple\"}"))
3636
val workflowType = Multipart.FormData.BodyPart("workflowType", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "WDL"))
3737
val workflowVersion = Multipart.FormData.BodyPart("workflowTypeVersion", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "1.0"))
38+
val workflowDependencies =
39+
Multipart.FormData.BodyPart(
40+
"workflowDependencies",
41+
HttpEntity(MediaTypes.`application/zip`, Array[Byte](0x0A, 0x0B, 0x0C)),
42+
)
3843

3944
val workflowSourceTriggerDescribeFailure =
4045
Multipart.FormData.BodyPart("workflowSource", HttpEntity(ContentTypes.`text/plain(UTF-8)`, "fail to describe"))
@@ -82,7 +87,8 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with
8287
"[reading back DescribeRequest contents] workflow url: None",
8388
"[reading back DescribeRequest contents] inputs: ",
8489
"[reading back DescribeRequest contents] type: None",
85-
"[reading back DescribeRequest contents] version: None"
90+
"[reading back DescribeRequest contents] version: None",
91+
"[reading back DescribeRequest contents] dependencies: None",
8692
),
8793
validWorkflow = true
8894
)
@@ -104,7 +110,8 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with
104110
"[reading back DescribeRequest contents] workflow url: Some(https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/wdl_draft3/valid/callable_imports/my_workflow.wdl)",
105111
"[reading back DescribeRequest contents] inputs: ",
106112
"[reading back DescribeRequest contents] type: None",
107-
"[reading back DescribeRequest contents] version: None"
113+
"[reading back DescribeRequest contents] version: None",
114+
"[reading back DescribeRequest contents] dependencies: None",
108115
),
109116
validWorkflow = true
110117
)
@@ -126,7 +133,40 @@ class WomtoolRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest with
126133
"[reading back DescribeRequest contents] workflow url: None",
127134
"[reading back DescribeRequest contents] inputs: {\"a\":\"is for apple\"}",
128135
"[reading back DescribeRequest contents] type: Some(WDL)",
129-
"[reading back DescribeRequest contents] version: Some(1.0)"
136+
"[reading back DescribeRequest contents] version: Some(1.0)",
137+
"[reading back DescribeRequest contents] dependencies: None",
138+
),
139+
validWorkflow = true
140+
)
141+
} { responseAs[WorkflowDescription] }
142+
}
143+
}
144+
145+
it should "include inputs, workflow type, workflow version, and workflow dependencies in the WorkflowSourceFilesCollection" in {
146+
Post(
147+
s"/womtool/$version/describe",
148+
Multipart.FormData(
149+
BodyParts.workflowSource,
150+
BodyParts.workflowInputs,
151+
BodyParts.workflowType,
152+
BodyParts.workflowVersion,
153+
BodyParts.workflowDependencies,
154+
).toEntity()
155+
) ~>
156+
akkaHttpService.womtoolRoutes ~>
157+
check {
158+
status should be(StatusCodes.OK)
159+
160+
assertResult {
161+
WorkflowDescription(valid = true,
162+
errors = List(
163+
"this is fake data from the mock SR actor",
164+
"[reading back DescribeRequest contents] workflow hashcode: Some(580529622)",
165+
"[reading back DescribeRequest contents] workflow url: None",
166+
"[reading back DescribeRequest contents] inputs: {\"a\":\"is for apple\"}",
167+
"[reading back DescribeRequest contents] type: Some(WDL)",
168+
"[reading back DescribeRequest contents] version: Some(1.0)",
169+
"[reading back DescribeRequest contents] dependencies: Some([0x0A, 0x0B, 0x0C])",
130170
),
131171
validWorkflow = true
132172
)

‎services/src/main/scala/cromwell/services/womtool/Describer.scala

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package cromwell.services.womtool
22

33
import cats.data.Validated.{Invalid, Valid}
4-
import cromwell.core.WorkflowSourceFilesCollection
4+
import cats.syntax.traverse._
5+
import common.validation.ErrorOr.ErrorOr
6+
import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection}
57
import cromwell.languages.util.ImportResolver.HttpResolver
68
import cromwell.languages.util.{ImportResolver, LanguageFactoryUtil}
79
import cromwell.languages.{LanguageFactory, ValidatedWomNamespace}
@@ -14,9 +16,22 @@ import wom.expression.NoIoFunctionSet
1416
object Describer {
1517

1618
def describeWorkflow(wsfc: WorkflowSourceFilesCollection): DescribeResult = {
19+
val zipResolverErrorOr: ErrorOr[Option[ImportResolver.ImportResolver]] =
20+
wsfc.importsZipFileOption.map(ImportResolver.zippedImportResolver(_, WorkflowId.randomId())).sequence
1721

18-
val initialResolvers = List(HttpResolver(None, Map.empty))
22+
val initialResolversErrorOr: ErrorOr[List[ImportResolver.ImportResolver]] =
23+
zipResolverErrorOr map { zipResolverOption =>
24+
zipResolverOption.toList ++ List(HttpResolver(None, Map.empty))
25+
}
26+
27+
initialResolversErrorOr match {
28+
case Valid(initialResolvers) => describeWorkflow(wsfc, initialResolvers)
29+
case Invalid(errors) => DescribeFailure(errors.toList.mkString(", "))
30+
}
31+
}
1932

33+
def describeWorkflow(wsfc: WorkflowSourceFilesCollection,
34+
initialResolvers: List[ImportResolver.ImportResolver]): DescribeResult = {
2035
// The HTTP resolver is used to pull down workflows submitted by URL
2136
LanguageFactoryUtil.findWorkflowSource(wsfc.workflowSource, wsfc.workflowUrl, initialResolvers) match {
2237
case Right((workflowSource: WorkflowSource, importResolvers: List[ImportResolver.ImportResolver])) =>

‎services/src/test/resources/describe/wdl_1_0/relative_imports/description.json

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"valid" : true,
3+
"errors" : [
4+
],
5+
"validWorkflow" : true,
6+
"name" : "relative_imports",
7+
"inputs" : [
8+
],
9+
"outputs" : [
10+
{
11+
"name" : "result",
12+
"valueType" : {
13+
"typeName" : "Int"
14+
},
15+
"typeDisplayName" : "Int"
16+
}
17+
],
18+
"images" : [
19+
],
20+
"submittedDescriptorType" : {
21+
"descriptorType" : "WDL",
22+
"descriptorTypeVersion" : "Biscayne"
23+
},
24+
"importedDescriptorTypes" : [
25+
],
26+
"meta" : {
27+
28+
},
29+
"parameterMeta" : {
30+
31+
},
32+
"isRunnableWorkflow" : true
33+
}

‎services/src/test/resources/describe/wdl_1_0/relative_imports/workflow.wdl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version development
2+
3+
import "sub_wfs/foo.wdl"
4+
5+
workflow relative_imports {
6+
call foo.foo_wf
7+
8+
output {
9+
Int result = foo_wf.unpacked
10+
}
11+
}

‎services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/structs/my_struct.wdl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
version development
2+
3+
struct MyStruct {
4+
Int a
5+
}

‎services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/sub_wfs/foo.wdl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version development
2+
3+
import "../structs/my_struct.wdl"
4+
import "tasks/add5.wdl" as a5
5+
6+
workflow foo_wf {
7+
call a5.add5 { input: x = object { a: 100 } }
8+
output {
9+
Int unpacked = add5.five_added.a
10+
}
11+
}

‎services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/sub_wfs/tasks/add5.wdl

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version development
2+
3+
import "../../structs/my_struct.wdl"
4+
5+
task add5 {
6+
input {
7+
MyStruct x
8+
}
9+
command <<<
10+
echo $((5 + ~{x.a}))
11+
>>>
12+
output {
13+
MyStruct five_added = object { a: read_int(stdout()) }
14+
}
15+
runtime {
16+
docker: "ubuntu:latest"
17+
}
18+
}

‎services/src/test/scala/cromwell/services/womtool/DescriberSpec.scala

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package cromwell.services.womtool
22

33
import common.assertion.CromwellTimeoutSpec
44
import cromwell.core.path._
5-
import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection, WorkflowSourceFilesWithoutImports}
5+
import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection}
66
import cromwell.languages.config.{CromwellLanguages, LanguageConfiguration}
77
import cromwell.services.womtool.DescriberSpec._
88
import cromwell.services.womtool.WomtoolServiceMessages.DescribeSuccess
@@ -37,26 +37,37 @@ class DescriberSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
3737
val workflowType = Try(caseDirectory.resolve("workflowType").contentAsString.stripLineEnd).toOption
3838
val workflowTypeVersion =
3939
Try(caseDirectory.resolve("workflowTypeVersion").contentAsString.stripLineEnd).toOption
40+
val importsFile =
41+
Try(caseDirectory.resolve("workflowDependencies")).filter(_.exists).map(_.zip()).toOption
4042

41-
val interimWsfc = WorkflowSourceFilesWithoutImports(
42-
workflowSource = None,
43-
workflowUrl = None,
43+
val workflowSource = testCase match {
44+
case FileAndDescription(file, _) => Option(file)
45+
case _ => None
46+
}
47+
48+
val workflowUrl = testCase match {
49+
case UrlAndDescription(url, _) => Option(url)
50+
case _ => None
51+
}
52+
53+
val wsfc = WorkflowSourceFilesCollection(
54+
workflowSource = workflowSource,
55+
workflowUrl = workflowUrl,
4456
workflowRoot = None,
4557
workflowType = workflowType,
4658
workflowTypeVersion = workflowTypeVersion,
4759
inputsJson = "",
4860
workflowOptions = WorkflowOptions.empty,
61+
importsFile = importsFile.map(_.byteArray),
62+
workflowOnHold = false,
4963
labelsJson = "",
5064
warnings = Seq.empty,
5165
requestedWorkflowId = None
5266
)
5367

54-
val wsfc = testCase match {
55-
case FileAndDescription(file, _) => interimWsfc.copy(workflowSource = Option(file))
56-
case UrlAndDescription(url, _) => interimWsfc.copy(workflowUrl = Option(url))
57-
}
58-
5968
check(wsfc, parse(testCase.expectedDescription).toOption.get)
69+
70+
importsFile.map(_.delete(swallowIOExceptions = true))
6071
}
6172
}
6273
}

0 commit comments

Comments
 (0)