Skip to content

Commit 126c374

Browse files
AN-381 Enable GCR mirroring of Dockerhub (#7740)
Co-authored-by: Adam Nichols <aednichols@gmail.com>
1 parent 5392f1b commit 126c374

File tree

29 files changed

+472
-26
lines changed

29 files changed

+472
-26
lines changed

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Cromwell Change Log
22

3+
## 90 Release Notes
4+
5+
### GCP Batch
6+
* Cromwell now supports automatic use of the [GAR Dockerhub mirror](https://cloud.google.com/artifact-registry/docs/pull-cached-dockerhub-images), see [ReadTheDocs](https://cromwell.readthedocs.io/en/develop/backends/GCPBatch/) for details.
7+
38
## 89 Release Notes
49

510
### Improvements

backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import cromwell.core.CallOutputs
88
import cromwell.core.JobToken.JobTokenType
99
import cromwell.core.path.Path
1010
import cromwell.core.path.PathFactory.PathBuilders
11+
import cromwell.docker.DockerMirroring
1112
import net.ceedubs.ficus.Ficus._
1213
import wom.expression.{IoFunctionSet, NoIoFunctionSet}
1314
import wom.graph.CommandCallNode
@@ -164,6 +165,11 @@ trait BackendLifecycleActorFactory extends PlatformSpecific {
164165
initializationDataOption: Option[BackendInitializationData]
165166
): List[Any] = List.empty
166167

168+
/**
169+
* Returns a DockerMirror built based on backend configuration
170+
*/
171+
val dockerMirroring: Option[DockerMirroring] = None
172+
167173
/**
168174
* Allows Cromwell to self-identify which cloud it's running on for runtime attribute purposes
169175
*/
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: hello_dockermirror_gcpbatch
2+
testFormat: workflowsuccess
3+
# needs an alt if we're going to keep Docker image cache tests which I'm not sure we are
4+
backends: [GCPBATCHDockerMirror]
5+
6+
files {
7+
workflow: hello_dockermirror_gcpbatch/hello.wdl
8+
inputs: hello_dockermirror_gcpbatch/hello.inputs
9+
}
10+
11+
metadata {
12+
workflowName: wf_hello
13+
status: Succeeded
14+
"calls.wf_hello.hello.runtimeAttributes.docker": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
15+
"calls.wf_hello.hello.dockerImageUsed": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
16+
}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"wf_hello.hello.addressee": "m'Lord"
3+
}
4+
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
task hello {
2+
String addressee
3+
command {
4+
echo "Hello ${addressee}!"
5+
}
6+
output {
7+
String salutation = read_string(stdout())
8+
}
9+
runtime {
10+
backend: "GCPBATCHDockerMirror"
11+
docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
12+
}
13+
}
14+
15+
workflow wf_hello {
16+
call hello
17+
output {
18+
hello.salutation
19+
}
20+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: hello_dockermirror_papiv2
2+
testFormat: workflowsuccess
3+
# needs an alt if we're going to keep Docker image cache tests which I'm not sure we are
4+
backends: [Papiv2DockerMirror]
5+
6+
files {
7+
workflow: hello_dockermirror_papiv2/hello.wdl
8+
inputs: hello_dockermirror_papiv2/hello.inputs
9+
}
10+
11+
metadata {
12+
workflowName: wf_hello
13+
status: Succeeded
14+
"calls.wf_hello.hello.runtimeAttributes.docker": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
15+
"calls.wf_hello.hello.dockerImageUsed": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
16+
}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"wf_hello.hello.addressee": "m'Lord"
3+
}
4+
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
task hello {
2+
String addressee
3+
command {
4+
echo "Hello ${addressee}!"
5+
}
6+
output {
7+
String salutation = read_string(stdout())
8+
}
9+
runtime {
10+
backend: "Papiv2DockerMirror"
11+
docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
12+
}
13+
}
14+
15+
workflow wf_hello {
16+
call hello
17+
output {
18+
hello.salutation
19+
}
20+
}

dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ sealed trait DockerImageIdentifier {
1111
def reference: String
1212

1313
def swapReference(newReference: String): DockerImageIdentifier
14+
def swapHost(newHost: String): DockerImageIdentifier
1415

1516
// The name of the image with a repository prefix iff a repository was explicitly specified.
1617
lazy val name = repository map { r => s"$r/$image" } getOrElse image
@@ -35,6 +36,7 @@ case class DockerImageIdentifierWithoutHash(host: Option[String],
3536
) extends DockerImageIdentifier {
3637
def withHash(hash: DockerHashResult) = DockerImageIdentifierWithHash(host, repository, image, reference, hash)
3738
override def swapReference(newReference: String) = this.copy(reference = newReference)
39+
override def swapHost(newHost: String) = this.copy(host = Option(newHost))
3840
}
3941

4042
case class DockerImageIdentifierWithHash(host: Option[String],
@@ -45,6 +47,7 @@ case class DockerImageIdentifierWithHash(host: Option[String],
4547
) extends DockerImageIdentifier {
4648
override lazy val fullName: String = s"$hostAsString$name@${hash.algorithmAndHash}"
4749
override def swapReference(newReference: String) = this.copy(reference = newReference)
50+
override def swapHost(newHost: String) = this.copy(host = Option(newHost))
4851
}
4952

5053
object DockerImageIdentifier {
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package cromwell.docker
2+
3+
import com.typesafe.config.Config
4+
import com.typesafe.scalalogging.LazyLogging
5+
import cromwell.docker.registryv2.flows.dockerhub.DockerHub
6+
import net.ceedubs.ficus.Ficus._
7+
import net.ceedubs.ficus.readers.ValueReader
8+
9+
case class DockerMirroring(mirrors: List[DockerMirror]) {
10+
def mirrorImage(image: DockerImageIdentifier): Option[DockerImageIdentifier] =
11+
mirrors.flatMap(_.mirrorImage.lift(image)).headOption
12+
}
13+
14+
object DockerMirroring {
15+
def fromConfig(config: Config): Option[DockerMirroring] = {
16+
val mirrors = config
17+
.getAs[Config]("docker-mirror")
18+
.map { mirrorConfig =>
19+
val dockerhubMirror = mirrorConfig.getAs[DockerHubMirror]("dockerhub")
20+
// Add support for additional repositories here
21+
22+
dockerhubMirror.toList
23+
}
24+
.getOrElse(List[DockerMirror]())
25+
Option.when(mirrors.nonEmpty)(DockerMirroring(mirrors))
26+
}
27+
}
28+
29+
sealed trait DockerMirror {
30+
val mirrorImage: PartialFunction[DockerImageIdentifier, DockerImageIdentifier]
31+
}
32+
33+
case class DockerHubMirror(address: String) extends DockerMirror {
34+
val mirrorImage: PartialFunction[DockerImageIdentifier, DockerImageIdentifier] = {
35+
case i if DockerHub.isValidDockerHubHost(i.host) => i.swapHost(address)
36+
}
37+
}
38+
39+
object DockerHubMirror extends LazyLogging {
40+
implicit val dockerHubMirrorOptionValueReader: ValueReader[Option[DockerHubMirror]] =
41+
(config: Config, path: String) =>
42+
config.getAs[Config](path) flatMap { dockerMirrorConfig =>
43+
val enabled = dockerMirrorConfig.as[Boolean]("enabled")
44+
val address = dockerMirrorConfig.as[Option[String]]("address").getOrElse("")
45+
if (address.isEmpty)
46+
logger.warn(
47+
"Potential misconfiguration: docker-mirror.dockerhub.enabled=true with no address provided. " +
48+
"Mirroring will be disabled."
49+
)
50+
Option.when(enabled && address.nonEmpty)(DockerHubMirror(address))
51+
}
52+
}

0 commit comments

Comments
 (0)