Replace URLConnection with HttpClient (#6188)

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>

Author: pditommaso

modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy

@@ -16,6 +16,7 @@
 
 package nextflow.scm
 
+import java.net.http.HttpResponse
 import java.util.regex.Pattern
 
 import groovy.transform.CompileDynamic
@@ -162,14 +163,11 @@ final class AzureRepositoryProvider extends RepositoryProvider {
      *
      * @param connection A {@link HttpURLConnection} connection instance
      */
-    protected checkResponse( HttpURLConnection connection ) {
-
-        if (connection.getHeaderFields().containsKey("x-ms-continuationtoken")) {
-            this.continuationToken = connection.getHeaderField("x-ms-continuationtoken");
-        } else {
-            this.continuationToken = null
-        }
-
+    protected checkResponse( HttpResponse<String> connection ) {
+        this.continuationToken = connection
+                .headers()
+                .firstValue("x-ms-continuationtoken")
+                .orElse(null)
         super.checkResponse(connection)
     }
 

modules/nextflow/src/main/groovy/nextflow/scm/GiteaRepositoryProvider.groovy

@@ -16,9 +16,9 @@
 
 package nextflow.scm
 
+
 import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
-
 /**
  * Implements a repository provider for Gitea service
  *
@@ -43,14 +43,14 @@ final class GiteaRepositoryProvider extends RepositoryProvider {
     }
 
     @Override
-    protected void auth( URLConnection connection ) {
+    protected String[] getAuth() {
         if( config.token ) {
             // set the token in the request header
             // https://docs.gitea.io/en-us/api-usage/#authentication
-            connection.setRequestProperty("Authorization", "token $config.token")
+            return new String[] { "Authorization", "token $config.token" as String }
         }
         else {
-            super.auth(connection)
+            return EMPTY_ARRAY
         }
     }
 

modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy

@@ -39,13 +39,15 @@ class GitlabRepositoryProvider extends RepositoryProvider {
     }
 
     @Override
-    protected void auth( URLConnection connection ) {
+    protected String[] getAuth() {
         if( config.token ) {
             // set the token in the request header
-            connection.setRequestProperty("PRIVATE-TOKEN", config.token)
-        } else if( config.password ) {
-            connection.setRequestProperty("PRIVATE-TOKEN", config.password)
+            return new String[] { "PRIVATE-TOKEN", config.token }
         }
+        if( config.password ) {
+            return new String[] { "PRIVATE-TOKEN", config.password }
+        }
+        return EMPTY_ARRAY
     }
 
     @Override

modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy

@@ -18,6 +18,12 @@ package nextflow.scm
 
 import static nextflow.util.StringUtils.*
 
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+import java.time.Duration
+import java.util.concurrent.Executors
+
 import groovy.json.JsonSlurper
 import groovy.transform.Canonical
 import groovy.transform.CompileStatic
@@ -26,6 +32,7 @@ import groovy.util.logging.Slf4j
 import nextflow.Const
 import nextflow.exception.AbortOperationException
 import nextflow.exception.RateLimitExceededException
+import nextflow.util.Threads
 import org.eclipse.jgit.api.Git
 import org.eclipse.jgit.lib.Ref
 import org.eclipse.jgit.transport.CredentialsProvider
@@ -40,6 +47,8 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider
 @CompileStatic
 abstract class RepositoryProvider {
 
+    static final public String[] EMPTY_ARRAY = new String[0]
+
     @Canonical
     static class TagInfo {
         String name
@@ -52,6 +61,11 @@ abstract class RepositoryProvider {
         String commitId
     }
 
+    /**
+     * The client used to carry out http requests
+     */
+    private HttpClient httpClient = { newHttpClient() }()
+
     /**
      * The pipeline qualified name following the syntax {@code owner/repository}
      */
@@ -184,24 +198,17 @@ abstract class RepositoryProvider {
         assert api
 
         log.debug "Request [credentials ${getAuthObfuscated() ?: '-'}] -> $api"
-        def connection = new URL(api).openConnection() as URLConnection
-        connection.setConnectTimeout(60_000)
-
-        auth(connection)
-
-        if( connection instanceof HttpURLConnection ) {
-            checkResponse(connection)
-        }
-
-        InputStream content = connection.getInputStream()
-        try {
-            final result = content.text
-            log.trace "Git provider HTTP request: '$api' -- Response:\n${result}"
-            return result
-        }
-        finally{
-            content?.close()
-        }
+        final request = HttpRequest
+            .newBuilder()
+            .uri(new URI(api))
+            .headers(getAuth())
+            .GET()
+        // submit the request
+        final resp = httpClient.send(request.build(), HttpResponse.BodyHandlers.ofString())
+        // check the response code
+        checkResponse(resp)
+        // return the body as string
+        return resp.body()
     }
 
     protected String getAuthObfuscated() {
@@ -211,15 +218,20 @@ abstract class RepositoryProvider {
     }
 
     /**
-     * Sets the authentication credential on the connection object
+     * Define the credentials to be used to authenticate the http request
      *
-     * @param connection The URL connection object to be authenticated
+     * @return
+     *      A string array holding the authentication HTTP headers e.g.
+     *      {@code [ "Authorization", "Bearer 1234567890"] } or an empty
+     *      array when the credentials are not available or provided.
+     *      Note: {@code null} is not a valid return value for this method.
      */
-    protected void auth( URLConnection connection ) {
+    protected String[] getAuth() {
         if( hasCredentials() ) {
             String authString = "${getUser()}:${getPassword()}".bytes.encodeBase64().toString()
-            connection.setRequestProperty("Authorization","Basic " + authString)
+            return new String[] { "Authorization", "Basic " + authString }
         }
+        return EMPTY_ARRAY
     }
 
     /**
@@ -228,17 +240,17 @@ abstract class RepositoryProvider {
      *
      * @param connection A {@link HttpURLConnection} connection instance
      */
-    protected checkResponse( HttpURLConnection connection ) {
-        def code = connection.getResponseCode()
+    protected checkResponse( HttpResponse<String> connection ) {
+        final code = connection.statusCode()
 
         switch( code ) {
             case 401:
-                log.debug "Response status: $code -- ${connection.getErrorStream()?.text}"
+                log.debug "Response status: $code -- ${connection.body()}"
                 throw new AbortOperationException("Not authorized -- Check that the ${name.capitalize()} user name and password provided are correct")
 
             case 403:
-                log.debug "Response status: $code -- ${connection.getErrorStream()?.text}"
-                def limit = connection.getHeaderField('X-RateLimit-Remaining')
+                log.debug "Response status: $code -- ${connection.body()}"
+                def limit = connection.headers().firstValue('X-RateLimit-Remaining').orElse(null)
                 if( limit == '0' ) {
                     def message = config.auth ? "Check ${name.capitalize()}'s API rate limits for more details" : "Provide your ${name.capitalize()} user name and password to get a higher rate limit"
                     throw new RateLimitExceededException("API rate limit exceeded -- $message")
@@ -248,8 +260,8 @@ abstract class RepositoryProvider {
                     throw new AbortOperationException("Forbidden -- $message")
                 }
             case 404:
-                log.debug "Response status: $code -- ${connection.getErrorStream()?.text}"
-                throw new AbortOperationException("Remote resource not found: ${connection.getURL()}")
+                log.debug "Response status: $code -- ${connection.body()}"
+                throw new AbortOperationException("Remote resource not found: ${connection.uri()}")
         }
     }
 
@@ -342,4 +354,15 @@ abstract class RepositoryProvider {
         }
     }
 
+    private HttpClient newHttpClient() {
+        final builder = HttpClient.newBuilder()
+            .version(HttpClient.Version.HTTP_1_1)
+            .connectTimeout(Duration.ofSeconds(60))
+        // use virtual threads executor if enabled
+        if( Threads.useVirtual() )
+            builder.executor(Executors.newVirtualThreadPerTaskExecutor())
+        // build and return the new client
+        return builder.build()
+    }
+
 }

modules/nextflow/src/test/groovy/nextflow/scm/GithubRepositoryProviderTest.groovy

@@ -126,17 +126,15 @@ class GithubRepositoryProviderTest extends Specification {
         SysEnv.push(['GITHUB_TOKEN': '1234567890'])
         and:
         def provider = Spy(new GithubRepositoryProvider('foo',Mock(ProviderConfig)))
-        and:
-        def conn = Mock(HttpURLConnection)
 
         when:
-        provider.auth(conn)
+        final result = provider.getAuth()
         then:
         _ * provider.getUser() 
         _ * provider.getPassword()
         1 * provider.hasCredentials()
         and:
-        1 * conn.setRequestProperty('Authorization', "Basic ${'1234567890:x-oauth-basic'.bytes.encodeBase64()}".toString())
+        result == new String[] { 'Authorization', "Basic ${'1234567890:x-oauth-basic'.bytes.encodeBase64()}".toString() }
 
         cleanup:
         SysEnv.pop()

modules/nextflow/src/test/groovy/nextflow/scm/RepositoryProviderTest.groovy

@@ -93,19 +93,21 @@ class RepositoryProviderTest extends Specification {
         def conn = Mock(HttpURLConnection)
 
         when:
-        provider.auth(conn)
+        def headers = provider.getAuth()
         then:
         1 * provider.getUser() >> null
         1 * provider.hasCredentials()
         0 * conn.setRequestProperty('Authorization', _)
+        and:
+        headers == [] as String[]
 
         when:
-        provider.auth(conn)
+        headers = provider.getAuth()
         then:
         _ * provider.getUser() >> 'foo'
         _ * provider.getPassword() >> 'bar'
         1 * provider.hasCredentials()
         and:
-        1 * conn.setRequestProperty('Authorization', "Basic ${'foo:bar'.bytes.encodeBase64()}")
+        headers == new String[] { 'Authorization', "Basic ${'foo:bar'.bytes.encodeBase64()}" }
     }
 }