Update nf-build-plugin gradle plugin to AWS SDK V2

buildSrc/build.gradle

@@ -7,12 +7,14 @@ repositories {
     mavenCentral()
 }
 
-version = "1.0.1"
+version = "2.0.0"
 group = "io.nextflow"
 
 dependencies {
-    implementation ('com.amazonaws:aws-java-sdk-s3:1.12.777')
     implementation 'com.google.code.gson:gson:2.13.1'
+    implementation 'software.amazon.awssdk:s3:2.31.64'
+    implementation 'software.amazon.awssdk:s3-transfer-manager:2.31.64'
+    implementation 'commons-codec:commons-codec:1.18.0'
 }
 
 gradlePlugin {

buildSrc/src/main/groovy/io/nextflow/gradle/tasks/AbstractS3Task.groovy

@@ -1,15 +1,11 @@
 package io.nextflow.gradle.tasks
 
-import com.amazonaws.auth.AWSCredentialsProviderChain
-import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider
-import com.amazonaws.auth.SystemPropertiesCredentialsProvider
-import com.amazonaws.auth.profile.ProfileCredentialsProvider
-import com.amazonaws.regions.Region
-import com.amazonaws.regions.Regions
-import com.amazonaws.services.s3.AmazonS3Client
 import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Internal
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
+import software.amazon.awssdk.services.s3.S3AsyncClient
+import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.regions.Region
 
 /**
  * S3 common operations
@@ -21,27 +17,32 @@ import org.gradle.api.tasks.Internal
 abstract class AbstractS3Task extends DefaultTask {
 
     @Internal
-    AmazonS3Client getS3Client() {
-        def profileCreds
+    S3Client getS3Client() {
+        def credBuilder = DefaultCredentialsProvider.builder()
         if (project.s3.profile) {
             logger.quiet("Using AWS credentials profile: ${project.s3.profile}")
-            profileCreds = new ProfileCredentialsProvider(project.s3.profile)
+            credBuilder.profileName(project.s3.profile)
         }
-        else {
-            profileCreds = new ProfileCredentialsProvider()
+        def clientBuilder = S3Client.builder().credentialsProvider(credBuilder.build())
+        String region = project.s3.region
+        if (region) {
+            clientBuilder.region(Region.of(region))
         }
-        def creds = new AWSCredentialsProviderChain(
-                new EnvironmentVariableCredentialsProvider(),
-                new SystemPropertiesCredentialsProvider(),
-                profileCreds,
-                new EC2ContainerCredentialsProviderWrapper()
-        )
+        return clientBuilder.build()
+    }
 
-        AmazonS3Client s3Client = new AmazonS3Client(creds)
+    @Internal
+    S3AsyncClient getS3AsyncClient() {
+        def credBuilder = DefaultCredentialsProvider.builder()
+        if (project.s3.profile) {
+            logger.quiet("Using AWS credentials profile: ${project.s3.profile}")
+            credBuilder.profileName(project.s3.profile)
+        }
+        def clientBuilder = S3AsyncClient.crtBuilder().credentialsProvider(credBuilder.build())
         String region = project.s3.region
         if (region) {
-            s3Client.region = Region.getRegion(Regions.fromName(region))
+            clientBuilder.region(Region.of(region))
         }
-        return s3Client
+        return clientBuilder.build()
     }
 }

buildSrc/src/main/groovy/io/nextflow/gradle/tasks/S3Download.groovy

@@ -1,16 +1,19 @@
 package io.nextflow.gradle.tasks
 
-import java.text.DecimalFormat
+import org.gradle.api.GradleException
 
-import com.amazonaws.event.ProgressEvent
-import com.amazonaws.event.ProgressListener
-import com.amazonaws.services.s3.transfer.Transfer
-import com.amazonaws.services.s3.transfer.TransferManager
+import java.text.DecimalFormat
 import groovy.transform.CompileStatic
 import io.nextflow.gradle.tasks.AbstractS3Task
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
+import software.amazon.awssdk.transfer.s3.S3TransferManager
+import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest
+import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest
+import software.amazon.awssdk.transfer.s3.progress.TransferListener
+
+import java.util.concurrent.ExecutionException
 
 /**
  * Download files from a S3 bucket
@@ -41,35 +44,70 @@ class S3Download extends AbstractS3Task {
 
     @TaskAction
     def task() {
-        TransferManager tm = new TransferManager(getS3Client())
-        Transfer transfer
+        def transferManager = S3TransferManager.builder()
+            .s3Client(getS3AsyncClient()) // ensure this returns an `S3AsyncClient`
+            .build()
+
+        def listener = new S3Listener()
 
         // directory download
         if (keyPrefix != null) {
             logger.quiet("S3 Download recursive s3://${bucket}/${keyPrefix} → ${project.file(destDir)}/")
-            transfer = (Transfer) tm.downloadDirectory(bucket, keyPrefix, project.file(destDir))
-        }
+            def request = DownloadDirectoryRequest.builder()
+                .bucket(bucket)
+                .listObjectsV2RequestTransformer(builder -> builder.prefix(keyPrefix))
+                .destination(project.file(destDir).toPath())
+                .downloadFileRequestTransformer(builder-> builder.addTransferListener(listener))
+                .build()
+
+            def download = transferManager.downloadDirectory(request)
+            try{
+			    def completed = download.completionFuture().get()
+                if (!completed.failedTransfers().isEmpty()){
+                    throw new GradleException("Some transfers in S3 download directory: s3://"+ bucket +"/"+ keyPrefix +" has failed - Transfers: " +  completed.failedTransfers() );
+			    }
+		    } catch (InterruptedException e){
+			    logger.debug("S3 download directory: s3://{}/{} interrupted", bucket, keyPrefix);
+			    Thread.currentThread().interrupt();
+		    } catch ( ExecutionException e) {
+                String msg = String.format("Exception thrown downloading S3 object s3://{}/{}", bucket, keyPrefix);
+                throw new GradleException(msg, e.getCause());
+            }
 
         // single file download
-        else {
+        } else {
             logger.quiet("S3 Download s3://${bucket}/${key} → ${file}")
             File f = new File(file)
             f.parentFile.mkdirs()
-            transfer = (Transfer) tm.download(bucket, key, f)
+            def request = DownloadFileRequest.builder()
+                .getObjectRequest { it.bucket(bucket).key(key) }
+                .destination(f.toPath())
+                .addTransferListener(listener)
+                .build()
+
+            def download = transferManager.downloadFile(request)
+            try{
+                download.completionFuture().get()
+            } catch (InterruptedException e){
+			    logger.debug("S3 download file: s3://{}/{} interrupted", bucket, key);
+			    Thread.currentThread().interrupt();
+		    } catch ( ExecutionException e) {
+                String msg = String.format("Exception thrown downloading S3 object s3://{}/{}", bucket, key);
+                throw new GradleException(msg, e.getCause());
+            }
         }
 
-        S3Listener listener = new S3Listener()
-        listener.transfer = transfer
-        transfer.addProgressListener(listener)
-        transfer.waitForCompletion()
+        transferManager.close()
     }
 
-    class S3Listener implements ProgressListener {
-        Transfer transfer
-
+    class S3Listener implements TransferListener {
         DecimalFormat df = new DecimalFormat("#0.0")
-        public void progressChanged(ProgressEvent e) {
-            logger.info("${df.format(transfer.progress.percentTransferred)}%")
+
+        @Override
+        void bytesTransferred(Context.BytesTransferred context) {
+            def request = context.request() as DownloadFileRequest
+            if (!context.progressSnapshot().ratioTransferred().empty)
+                logger.info("${request.destination()}: ${df.format(context.progressSnapshot().ratioTransferred().asDouble)}%")
         }
     }
 }

buildSrc/src/main/groovy/io/nextflow/gradle/tasks/S3Upload.groovy

@@ -1,7 +1,5 @@
 package io.nextflow.gradle.tasks
 
-import com.amazonaws.services.s3.model.CannedAccessControlList
-import com.amazonaws.services.s3.model.PutObjectRequest
 import groovy.transform.CompileStatic
 import io.nextflow.gradle.tasks.AbstractS3Task
 import io.nextflow.gradle.util.BucketTokenizer
@@ -10,6 +8,13 @@ import org.gradle.api.GradleException
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
+import software.amazon.awssdk.services.s3.model.GetObjectRequest
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException
+import software.amazon.awssdk.services.s3.model.ObjectCannedACL
+import software.amazon.awssdk.services.s3.model.PutObjectRequest
+import software.amazon.awssdk.services.s3.model.S3Exception
+
 /**
  * Upload files to an S3 bucket
  * 
@@ -56,7 +61,22 @@ class S3Upload extends AbstractS3Task {
         if( !sourceFile.exists() )
             throw new GradleException("S3 upload failed -- source file does not exist: $sourceFile")
 
-        if (s3Client.doesObjectExist(bucket, targetKey)) {
+        boolean objectExists = false
+        try {
+            s3Client.headObject(HeadObjectRequest.builder()
+                .bucket(bucket)
+                .key(targetKey)
+                .build()
+            )
+            objectExists = true
+        } catch ( NoSuchKeyException | S3Exception e) {
+            if (e.awsErrorDetails()?.errorCode() == 'NotFound') {
+                objectExists = false
+            } else {
+                throw new GradleException("S3 upload failed -- error checking if object exists: ${e.message}", e)
+            }
+        }
+        if (objectExists) {
             if( skipExisting ) {
                 logger.quiet("s3://${bucket}/${targetKey} already exists -- skipping")
             }
@@ -79,10 +99,14 @@ class S3Upload extends AbstractS3Task {
     boolean isSameContent(File sourceFile, String bucket, String targetKey) {
         final d1 = sourceFile
                 .withInputStream { InputStream it -> DigestUtils.sha512Hex(it) }
-        final d2 = s3Client
-                .getObject(bucket, targetKey)
-                .getObjectContent()
-                .withStream { InputStream it -> DigestUtils.sha512Hex(it) }
+
+        final request = GetObjectRequest.builder()
+            .bucket(bucket)
+            .key(targetKey)
+            .build()
+
+        final d2 = s3Client.getObject(request)
+            .withStream { InputStream it -> DigestUtils.sha512Hex(it) }
         return d1 == d2
     }
 
@@ -91,12 +115,12 @@ class S3Upload extends AbstractS3Task {
             logger.quiet("S3 will upload ${sourceFile} → s3://${bucket}/${targetKey} ${exists ? '[would overwrite existing]' : ''}")
         }
         else {
-            final req = new PutObjectRequest(bucket, targetKey, sourceFile)
+            final builder = PutObjectRequest.builder().bucket(bucket).key(targetKey)
             if( publicRead )
-                req.withCannedAcl(CannedAccessControlList.PublicRead)
+                builder.acl(ObjectCannedACL.PUBLIC_READ)
 
             logger.quiet("S3 upload ${sourceFile} → s3://${bucket}/${targetKey} ${exists ? '[overwrite existing]': ''}")
-            s3Client.putObject(req)
+            s3Client.putObject(builder.build(), sourceFile.toPath())
         }
     }
 }