AN-532 Finite DRS timeout (#7736)

Author: aednichols

centaur/src/main/scala/centaur/test/CentaurTestException.scala

@@ -3,6 +3,8 @@ package centaur.test
 import centaur.test.workflow.Workflow
 import cromwell.api.model.{SubmittedWorkflow, WorkflowMetadata}
 
+import scala.util.control.NoStackTrace
+
 /**
   * An exception with information about a centaur test failure.
   *
@@ -18,6 +20,7 @@ case class CentaurTestException private (message: String,
                                          metadataJsonOption: Option[String],
                                          causeOption: Option[Exception]
 ) extends RuntimeException(message, causeOption.orNull)
+    with NoStackTrace
 
 object CentaurTestException {
 

cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala

@@ -7,6 +7,7 @@ import scala.concurrent.duration._
 import scala.language.postfixOps
 
 final case class DrsConfig(drsResolverUrl: String,
+                           requestTimeout: FiniteDuration,
                            numRetries: Int,
                            waitInitial: FiniteDuration,
                            waitMaximum: FiniteDuration,
@@ -16,13 +17,15 @@ final case class DrsConfig(drsResolverUrl: String,
 
 object DrsConfig {
   // If you update these values also update Filesystems.md!
+  private val DefaultRequestTimeout = 60 seconds
   private val DefaultNumRetries = 3
   private val DefaultWaitInitial = 30 seconds
   private val DefaultWaitMaximum = 60 seconds
   private val DefaultWaitMultiplier = 1.25d
   private val DefaultWaitRandomizationFactor = 0.1
 
   private val EnvDrsResolverUrl = "DRS_RESOLVER_URL"
+  private val EnvDrsResolverRequestTimeout = "DRS_RESOLVER_REQUEST_TIMEOUT"
   private val EnvDrsResolverNumRetries = "DRS_RESOLVER_NUM_RETRIES"
   private val EnvDrsResolverWaitInitialSeconds = "DRS_RESOLVER_WAIT_INITIAL_SECONDS"
   private val EnvDrsResolverWaitMaximumSeconds = "DRS_RESOLVER_WAIT_MAXIMUM_SECONDS"
@@ -32,6 +35,7 @@ object DrsConfig {
   def fromConfig(drsResolverConfig: Config): DrsConfig =
     DrsConfig(
       drsResolverUrl = drsResolverConfig.getString("url"),
+      requestTimeout = drsResolverConfig.getOrElse("request-timeout", DefaultRequestTimeout),
       numRetries = drsResolverConfig.getOrElse("num-retries", DefaultNumRetries),
       waitInitial = drsResolverConfig.getOrElse("wait-initial", DefaultWaitInitial),
       waitMaximum = drsResolverConfig.getOrElse("wait-maximum", DefaultWaitMaximum),
@@ -42,6 +46,7 @@ object DrsConfig {
   def fromEnv(env: Map[String, String]): DrsConfig =
     DrsConfig(
       drsResolverUrl = env(EnvDrsResolverUrl),
+      requestTimeout = env.get(EnvDrsResolverRequestTimeout).map(_.toLong.seconds).getOrElse(DefaultRequestTimeout),
       numRetries = env.get(EnvDrsResolverNumRetries).map(_.toInt).getOrElse(DefaultNumRetries),
       waitInitial = env.get(EnvDrsResolverWaitInitialSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitInitial),
       waitMaximum = env.get(EnvDrsResolverWaitMaximumSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitMaximum),
@@ -53,6 +58,7 @@ object DrsConfig {
   def toEnv(drsConfig: DrsConfig): Map[String, String] =
     Map(
       EnvDrsResolverUrl -> drsConfig.drsResolverUrl,
+      EnvDrsResolverRequestTimeout -> s"${drsConfig.requestTimeout.toSeconds}",
       EnvDrsResolverNumRetries -> s"${drsConfig.numRetries}",
       EnvDrsResolverWaitInitialSeconds -> s"${drsConfig.waitInitial.toSeconds}",
       EnvDrsResolverWaitMaximumSeconds -> s"${drsConfig.waitMaximum.toSeconds}",

cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala

@@ -14,6 +14,7 @@ import io.circe.parser.decode
 import io.circe.syntax._
 import mouse.boolean._
 import org.apache.commons.lang3.exception.ExceptionUtils
+import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.{HttpGet, HttpPost}
 import org.apache.http.entity.{ContentType, StringEntity}
 import org.apache.http.impl.client.HttpClientBuilder
@@ -61,6 +62,7 @@ class DrsPathResolver(drsConfig: DrsConfig, drsCredentials: DrsCredentials) {
           postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON))
           postRequest.setHeader("Authorization", s"Bearer $token")
           postRequest.setHeader("X-App-ID", "cromwell_drs_localizer")
+          postRequest.setConfig(timeoutConfig)
           postRequest
         }
       case Invalid(errors) =>
@@ -69,6 +71,17 @@ class DrsPathResolver(drsConfig: DrsConfig, drsCredentials: DrsCredentials) {
     Resource.eval(io)
   }
 
+  private lazy val timeoutConfig: RequestConfig = {
+    // Infinite timeout bad, can exhaust Akka threads. [AN-532]
+    val requestTimeoutMillis = drsConfig.requestTimeout.toMillis.toInt
+    RequestConfig
+      .custom()
+      .setConnectionRequestTimeout(requestTimeoutMillis / 4) // Obtain connection from the pool
+      .setConnectTimeout(requestTimeoutMillis) // Max time to first byte from server
+      .setSocketTimeout(requestTimeoutMillis / 4) // Max time between subsequent bytes
+      .build()
+  }
+
   private def httpResponseToDrsResolverResponse(
     drsPathForDebugging: String
   )(httpResponse: HttpResponse): IO[DrsResolverResponse] = {

cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala

@@ -1,11 +1,12 @@
 package cloud.nio.impl.drs
 
 import scala.concurrent.duration._
+import scala.language.postfixOps
 
 object MockDrsPaths {
   val drsResolverUrl = "http://mock.drshub"
 
-  val mockDrsConfig: DrsConfig = DrsConfig(MockDrsPaths.drsResolverUrl, 1, 1.seconds, 1.seconds, 1d, 0d)
+  val mockDrsConfig: DrsConfig = DrsConfig(MockDrsPaths.drsResolverUrl, 5 seconds, 1, 1 seconds, 1 seconds, 1d, 0d)
 
   val mockToken = "mock.token"
 

docs/filesystems/Filesystems.md

@@ -20,6 +20,9 @@ filesystems {
         config {
           resolver {
             url = https://drshub-url-here"
+            # How long to wait for the server to start responding
+            # Default is the informal standard around the DRS ecosystem
+            request-timeout = 60 seconds
             # The number of times to retry failures connecting or HTTP 429 or HTTP 5XX responses, default 3.
             num-retries = 3
             # How long to wait between retrying HTTP 429 or HTTP 5XX responses, default 10 seconds.