AN-381 Enable GCR mirroring of Dockerhub (#7740)

Co-authored-by: Adam Nichols <aednichols@gmail.com>

Author: jgainerdewar

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Cromwell Change Log
 
+## 90 Release Notes
+
+### GCP Batch
+ * Cromwell now supports automatic use of the [GAR Dockerhub mirror](https://cloud.google.com/artifact-registry/docs/pull-cached-dockerhub-images), see [ReadTheDocs](https://cromwell.readthedocs.io/en/develop/backends/GCPBatch/) for details.
+
 ## 89 Release Notes
 
 ### Improvements

backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala

@@ -8,6 +8,7 @@ import cromwell.core.CallOutputs
 import cromwell.core.JobToken.JobTokenType
 import cromwell.core.path.Path
 import cromwell.core.path.PathFactory.PathBuilders
+import cromwell.docker.DockerMirroring
 import net.ceedubs.ficus.Ficus._
 import wom.expression.{IoFunctionSet, NoIoFunctionSet}
 import wom.graph.CommandCallNode
@@ -164,6 +165,11 @@ trait BackendLifecycleActorFactory extends PlatformSpecific {
                             initializationDataOption: Option[BackendInitializationData]
   ): List[Any] = List.empty
 
+  /**
+   * Returns a DockerMirror built based on backend configuration
+   */
+  val dockerMirroring: Option[DockerMirroring] = None
+
   /**
     * Allows Cromwell to self-identify which cloud it's running on for runtime attribute purposes
     */

centaur/src/main/resources/standardTestCases/hello_dockermirror_gcpbatch.test

@@ -0,0 +1,16 @@
+name: hello_dockermirror_gcpbatch
+testFormat: workflowsuccess
+# needs an alt if we're going to keep Docker image cache tests which I'm not sure we are
+backends: [GCPBATCHDockerMirror]
+
+files {
+  workflow: hello_dockermirror_gcpbatch/hello.wdl
+  inputs: hello_dockermirror_gcpbatch/hello.inputs
+}
+
+metadata {
+  workflowName: wf_hello
+  status: Succeeded
+  "calls.wf_hello.hello.runtimeAttributes.docker": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+  "calls.wf_hello.hello.dockerImageUsed": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+}

centaur/src/main/resources/standardTestCases/hello_dockermirror_gcpbatch/hello.inputs

@@ -0,0 +1,4 @@
+{
+  "wf_hello.hello.addressee": "m'Lord"
+}
+

centaur/src/main/resources/standardTestCases/hello_dockermirror_gcpbatch/hello.wdl

@@ -0,0 +1,20 @@
+task hello {
+  String addressee
+  command {
+    echo "Hello ${addressee}!"
+  }
+  output {
+    String salutation = read_string(stdout())
+  }
+  runtime {
+    backend: "GCPBATCHDockerMirror"
+    docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+  }
+}
+
+workflow wf_hello {
+  call hello
+  output {
+     hello.salutation
+  }
+}

centaur/src/main/resources/standardTestCases/hello_dockermirror_papiv2.test

@@ -0,0 +1,16 @@
+name: hello_dockermirror_papiv2
+testFormat: workflowsuccess
+# needs an alt if we're going to keep Docker image cache tests which I'm not sure we are
+backends: [Papiv2DockerMirror]
+
+files {
+  workflow: hello_dockermirror_papiv2/hello.wdl
+  inputs: hello_dockermirror_papiv2/hello.inputs
+}
+
+metadata {
+  workflowName: wf_hello
+  status: Succeeded
+  "calls.wf_hello.hello.runtimeAttributes.docker": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+  "calls.wf_hello.hello.dockerImageUsed": "mirror.gcr.io/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+}

centaur/src/main/resources/standardTestCases/hello_dockermirror_papiv2/hello.inputs

@@ -0,0 +1,4 @@
+{
+  "wf_hello.hello.addressee": "m'Lord"
+}
+

centaur/src/main/resources/standardTestCases/hello_dockermirror_papiv2/hello.wdl

@@ -0,0 +1,20 @@
+task hello {
+  String addressee
+  command {
+    echo "Hello ${addressee}!"
+  }
+  output {
+    String salutation = read_string(stdout())
+  }
+  runtime {
+    backend: "Papiv2DockerMirror"
+    docker: "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+  }
+}
+
+workflow wf_hello {
+  call hello
+  output {
+     hello.salutation
+  }
+}

dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala

@@ -11,6 +11,7 @@ sealed trait DockerImageIdentifier {
   def reference: String
 
   def swapReference(newReference: String): DockerImageIdentifier
+  def swapHost(newHost: String): DockerImageIdentifier
 
   // The name of the image with a repository prefix iff a repository was explicitly specified.
   lazy val name = repository map { r => s"$r/$image" } getOrElse image
@@ -35,6 +36,7 @@ case class DockerImageIdentifierWithoutHash(host: Option[String],
 ) extends DockerImageIdentifier {
   def withHash(hash: DockerHashResult) = DockerImageIdentifierWithHash(host, repository, image, reference, hash)
   override def swapReference(newReference: String) = this.copy(reference = newReference)
+  override def swapHost(newHost: String) = this.copy(host = Option(newHost))
 }
 
 case class DockerImageIdentifierWithHash(host: Option[String],
@@ -45,6 +47,7 @@ case class DockerImageIdentifierWithHash(host: Option[String],
 ) extends DockerImageIdentifier {
   override lazy val fullName: String = s"$hostAsString$name@${hash.algorithmAndHash}"
   override def swapReference(newReference: String) = this.copy(reference = newReference)
+  override def swapHost(newHost: String) = this.copy(host = Option(newHost))
 }
 
 object DockerImageIdentifier {

dockerHashing/src/main/scala/cromwell/docker/DockerMirroring.scala

@@ -0,0 +1,52 @@
+package cromwell.docker
+
+import com.typesafe.config.Config
+import com.typesafe.scalalogging.LazyLogging
+import cromwell.docker.registryv2.flows.dockerhub.DockerHub
+import net.ceedubs.ficus.Ficus._
+import net.ceedubs.ficus.readers.ValueReader
+
+case class DockerMirroring(mirrors: List[DockerMirror]) {
+  def mirrorImage(image: DockerImageIdentifier): Option[DockerImageIdentifier] =
+    mirrors.flatMap(_.mirrorImage.lift(image)).headOption
+}
+
+object DockerMirroring {
+  def fromConfig(config: Config): Option[DockerMirroring] = {
+    val mirrors = config
+      .getAs[Config]("docker-mirror")
+      .map { mirrorConfig =>
+        val dockerhubMirror = mirrorConfig.getAs[DockerHubMirror]("dockerhub")
+        // Add support for additional repositories here
+
+        dockerhubMirror.toList
+      }
+      .getOrElse(List[DockerMirror]())
+    Option.when(mirrors.nonEmpty)(DockerMirroring(mirrors))
+  }
+}
+
+sealed trait DockerMirror {
+  val mirrorImage: PartialFunction[DockerImageIdentifier, DockerImageIdentifier]
+}
+
+case class DockerHubMirror(address: String) extends DockerMirror {
+  val mirrorImage: PartialFunction[DockerImageIdentifier, DockerImageIdentifier] = {
+    case i if DockerHub.isValidDockerHubHost(i.host) => i.swapHost(address)
+  }
+}
+
+object DockerHubMirror extends LazyLogging {
+  implicit val dockerHubMirrorOptionValueReader: ValueReader[Option[DockerHubMirror]] =
+    (config: Config, path: String) =>
+      config.getAs[Config](path) flatMap { dockerMirrorConfig =>
+        val enabled = dockerMirrorConfig.as[Boolean]("enabled")
+        val address = dockerMirrorConfig.as[Option[String]]("address").getOrElse("")
+        if (address.isEmpty)
+          logger.warn(
+            "Potential misconfiguration: docker-mirror.dockerhub.enabled=true with no address provided. " +
+              "Mirroring will be disabled."
+          )
+        Option.when(enabled && address.nonEmpty)(DockerHubMirror(address))
+      }
+}