Include support for git-remote-s3

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -307,7 +307,7 @@ class AssetManager {
     @PackageScope
     String resolveNameFromGitUrl( String repository ) {
 
-        final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/')
+        final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('s3://')
         if( !isUrl )
             return null
 

modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy

@@ -88,6 +88,7 @@ class RepositoryFactory implements ExtensionPoint {
 
     // --==  static definitions ==--
     private static boolean codeCommitLoaded
+    private static boolean s3Loaded
     private static List<RepositoryFactory> factories0
 
     private static List<RepositoryFactory> factories() {
@@ -102,6 +103,11 @@ class RepositoryFactory implements ExtensionPoint {
 
     static RepositoryProvider newRepositoryProvider(ProviderConfig config, String project) {
         // check if it's needed to load new plugins
+        if( config.platform == 's3' && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( (config.name=='codecommit' || config.platform=='codecommit') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true
@@ -120,6 +126,11 @@ class RepositoryFactory implements ExtensionPoint {
 
     static ProviderConfig newProviderConfig(String name, Map<String,Object> attrs) {
         // check if it's needed to load new plugins
+        if( attrs.platform == 's3' && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( (name=='codecommit' || attrs.platform=='codecommit') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true
@@ -134,6 +145,11 @@ class RepositoryFactory implements ExtensionPoint {
     }
 
     static ProviderConfig getProviderConfig(List<ProviderConfig> providers, GitUrl url) {
+        if( url.protocol.equals('s3') && !s3Loaded){
+            Plugins.startIfMissing('nf-amazon')
+            s3Loaded=true
+            factories0=null
+        }
         if( url.domain.startsWith('git-codecommit.') && url.domain.endsWith('.amazonaws.com') && !codeCommitLoaded ) {
             Plugins.startIfMissing('nf-codecommit')
             codeCommitLoaded=true

plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3ProviderConfig.groovy

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cloud.aws.scm
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.Global
+import nextflow.exception.AbortOperationException
+import nextflow.scm.ProviderConfig
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
+import software.amazon.awssdk.regions.Region
+
+/**
+ * Implements a provider config for git-remote-s3 repositories
+ *
+ * @author Jorge Ejarque <jorge.ejarque@seqera.io>
+ */
+@Slf4j
+@CompileStatic
+class S3ProviderConfig extends ProviderConfig {
+
+    private Region region = Region.US_EAST_1
+
+    private AwsCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.builder().build()
+
+    S3ProviderConfig(String name, Map values) {
+        super(name, values)
+        setDefaultsFromAwsConfig()
+        // Override with scm repo attributes
+        setValuesFromMap(values)
+    }
+
+    S3ProviderConfig(String name){
+        super(name,[ platform: 's3', server: "s3://$name"])
+        setDefaultsFromAwsConfig()
+    }
+
+    private void setDefaultsFromAwsConfig() {
+        final config = Global.session?.config?.aws as Map
+        if( config ) {
+            setValuesFromMap(config)
+        }
+    }
+    private void setValuesFromMap(Map values){
+        if( values.region )
+            region = Region.of(values.region as String)
+        if( values.accessKey && values.secretKey ){
+            awsCredentialsProvider = StaticCredentialsProvider.create(
+                AwsBasicCredentials.builder()
+                    .accessKeyId(values.accessKey as String)
+                    .secretAccessKey(values.secretKey as String)
+                    .build())
+        }
+    }
+
+    Region getRegion(){
+        this.region
+    }
+
+    AwsCredentialsProvider getAwsCredentialsProvider(){
+        this.awsCredentialsProvider
+    }
+
+    @Override
+    protected String resolveProjectName(String path){
+        log.debug ("Resolving project name from $path. returning ")
+        if (!server.startsWith('s3://'))
+            new AbortOperationException("S3 project server doesn't start with s3://")
+        return "${server.substring('s3://'.size())}/$path"
+    }
+
+}

plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryFactory.groovy

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cloud.aws.scm
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import nextflow.cloud.aws.scm.jgit.TransportS3
+import nextflow.plugin.Priority
+import nextflow.scm.GitUrl
+import nextflow.scm.ProviderConfig
+import nextflow.scm.RepositoryFactory
+import nextflow.scm.RepositoryProvider
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+/**
+ * Implements a factory to create an instance of {@link S3RepositoryProvider}
+ *
+ * @author Jorge Ejarque <jorge.ejarque@seqera.io>
+ */
+@Slf4j
+@Priority(-10)
+@CompileStatic
+class S3RepositoryFactory extends RepositoryFactory{
+
+    private static AtomicBoolean registered = new AtomicBoolean(false)
+
+    @Override
+    protected RepositoryProvider createProviderInstance(ProviderConfig config, String project) {
+        if (!registered.get()) {
+            registered.set(true)
+            TransportS3.register()
+        }
+
+        return config.platform == 's3'
+            ? new S3RepositoryProvider(project, config)
+            : null
+    }
+
+    @Override
+    protected ProviderConfig getConfig(List<ProviderConfig> providers, GitUrl url) {
+        // do not care about non AWS codecommit url
+        if( url.protocol != 's3' )
+            return null
+
+        // S3 repository config depends on the bucket name stored as domain
+        def config = providers.find( it -> it.domain == url.domain )
+        if( config ) {
+            log.debug "Git url=$url (1) -> config=$config"
+            return config
+        }
+
+        // still nothing, create a new instance
+        config = new S3ProviderConfig(url.domain)
+
+
+        return config
+    }
+
+    @Override
+    protected ProviderConfig createConfigInstance(String name, Map attrs) {
+        final copy = new HashMap(attrs)
+        return copy.platform == 's3'
+            ? new S3ProviderConfig(name, copy)
+            : null
+    }
+
+
+
+
+}

plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryProvider.groovy

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.cloud.aws.scm
+
+import groovy.transform.CompileStatic
+import groovy.transform.Memoized
+import groovy.util.logging.Slf4j
+import nextflow.cloud.aws.scm.jgit.S3GitCredentialsProvider
+import nextflow.exception.AbortOperationException
+import nextflow.scm.ProviderConfig
+import nextflow.scm.RepositoryProvider
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.api.errors.TransportException
+import org.eclipse.jgit.transport.CredentialsProvider
+
+import java.nio.file.Files
+
+
+/**
+ * Implements a repository provider for git-remote-s3 repositories.
+ *
+ * @author Jorge Ejarque <jorge.ejarque@seqera.io>
+ */
+@Slf4j
+@CompileStatic
+class S3RepositoryProvider extends RepositoryProvider {
+
+    S3RepositoryProvider(String project, ProviderConfig config) {
+        assert config instanceof S3ProviderConfig
+        log.debug("Creating S3 repository provider for $project")
+        this.project = project
+        this.config = config
+    }
+    /** {@inheritDoc} **/
+    @Memoized
+    @Override
+    CredentialsProvider getGitCredentials() {
+        final providerConfig = this.config as S3ProviderConfig
+        final credentials = new S3GitCredentialsProvider()
+        if( providerConfig.region )
+            credentials.setRegion(providerConfig.region)
+        if( providerConfig.awsCredentialsProvider )
+            credentials.setAwsCredentialsProvider(providerConfig.awsCredentialsProvider)
+        return credentials
+    }
+
+    /** {@inheritDoc} **/
+    // called by AssetManager
+    // used to set credentials for a clone, pull, fetch, operation
+    @Override
+    boolean hasCredentials() {
+        // set to true
+        // uses AWS Credentials instead of username : password
+        // see getGitCredentials()
+        return true
+    }
+
+    /** {@inheritDoc} **/
+    @Override
+    String getName() { return project }
+
+    /** {@inheritDoc} **/
+    @Override
+    String getEndpointUrl() {
+        return "s3://$project"
+    }
+
+    /** {@inheritDoc} **/
+    // not used, but the abstract method needs to be overridden
+    @Override
+    String getContentUrl( String path ) {
+        throw new UnsupportedOperationException()
+    }
+
+    /** {@inheritDoc} **/
+    // called by AssetManager
+    @Override
+    String getCloneUrl() { getEndpointUrl() }
+
+    /** {@inheritDoc} **/
+    // called by AssetManager
+    @Override
+    String getRepositoryUrl() { getEndpointUrl() }
+
+    /** {@inheritDoc} **/
+    // called by AssetManager
+    // called by RepositoryProvider.readText()
+    @Override
+    byte[] readBytes( String path ) {
+        log.debug("Reading $path")
+        //Not possible to get a single file requires to clone the branch and get the file
+        final tmpDir = Files.createTempDirectory("s3-git-remote")
+        final command = Git.cloneRepository()
+            .setURI(getEndpointUrl())
+            .setDirectory(tmpDir.toFile())
+            .setCredentialsProvider(getGitCredentials())
+        if( revision )
+            command.setBranch(revision)
+        try {
+            command.call()
+            final file = tmpDir.resolve(path)
+            return file.getBytes()
+        }
+        catch (Exception e) {
+            log.debug(" unable to retrieve file: $path from repo: $project", e)
+            return null
+        }
+        finally{
+            tmpDir.deleteDir()
+        }
+    }
+
+    /** {@inheritDoc} **/
+    // called by AssetManager
+    @Override
+    void validateRepo() {
+        // Nothing to check
+    }
+
+    private String errMsg(Exception e) {
+        def msg = "Unable to access Git repository"
+        if( e.message )
+            msg + " - ${e.message}"
+        else
+            msg += ": " + getCloneUrl()
+        return msg
+    }
+
+    @Override
+    List<BranchInfo> getBranches() {
+        try {
+            return super.getBranches()
+        }
+        catch ( TransportException e) {
+            throw new AbortOperationException(errMsg(e), e)
+        }
+    }
+
+    @Override
+    List<TagInfo> getTags() {
+        try {
+            return super.getTags()
+        }
+        catch (TransportException e) {
+            throw new AbortOperationException(errMsg(e), e)
+        }
+    }
+
+
+}