diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index b35f1a06d..f432b87fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -307,7 +307,7 @@ class AssetManager { @PackageScope String resolveNameFromGitUrl( String repository ) { - final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') + final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('s3://') if( !isUrl ) return null diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy index 0856ce24d..c305fee05 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy @@ -88,6 +88,7 @@ class RepositoryFactory implements ExtensionPoint { // --== static definitions ==-- private static boolean codeCommitLoaded + private static boolean s3Loaded private static List factories0 private static List factories() { @@ -102,6 +103,11 @@ class RepositoryFactory implements ExtensionPoint { static RepositoryProvider newRepositoryProvider(ProviderConfig config, String project) { // check if it's needed to load new plugins + if( config.platform == 's3' && !s3Loaded){ + Plugins.startIfMissing('nf-amazon') + s3Loaded=true + factories0=null + } if( (config.name=='codecommit' || config.platform=='codecommit') && !codeCommitLoaded ) { Plugins.startIfMissing('nf-codecommit') codeCommitLoaded=true @@ -120,6 +126,11 @@ class RepositoryFactory implements ExtensionPoint { static ProviderConfig newProviderConfig(String name, Map attrs) { // check if it's needed to load new plugins + if( attrs.platform == 's3' && !s3Loaded){ + Plugins.startIfMissing('nf-amazon') + s3Loaded=true + factories0=null + } if( (name=='codecommit' || attrs.platform=='codecommit') && !codeCommitLoaded ) { Plugins.startIfMissing('nf-codecommit') codeCommitLoaded=true @@ -134,6 +145,11 @@ class RepositoryFactory implements ExtensionPoint { } static ProviderConfig getProviderConfig(List providers, GitUrl url) { + if( url.protocol.equals('s3') && !s3Loaded){ + Plugins.startIfMissing('nf-amazon') + s3Loaded=true + factories0=null + } if( url.domain.startsWith('git-codecommit.') && url.domain.endsWith('.amazonaws.com') && !codeCommitLoaded ) { Plugins.startIfMissing('nf-codecommit') codeCommitLoaded=true diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3ProviderConfig.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3ProviderConfig.groovy new file mode 100644 index 000000000..2d66a28d0 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3ProviderConfig.groovy @@ -0,0 +1,89 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Global +import nextflow.exception.AbortOperationException +import nextflow.scm.ProviderConfig +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.regions.Region + +/** + * Implements a provider config for git-remote-s3 repositories + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class S3ProviderConfig extends ProviderConfig { + + private Region region = Region.US_EAST_1 + + private AwsCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.builder().build() + + S3ProviderConfig(String name, Map values) { + super(name, values) + setDefaultsFromAwsConfig() + // Override with scm repo attributes + setValuesFromMap(values) + } + + S3ProviderConfig(String name){ + super(name,[ platform: 's3', server: "s3://$name"]) + setDefaultsFromAwsConfig() + } + + private void setDefaultsFromAwsConfig() { + final config = Global.session?.config?.aws as Map + if( config ) { + setValuesFromMap(config) + } + } + private void setValuesFromMap(Map values){ + if( values.region ) + region = Region.of(values.region as String) + if( values.accessKey && values.secretKey ){ + awsCredentialsProvider = StaticCredentialsProvider.create( + AwsBasicCredentials.builder() + .accessKeyId(values.accessKey as String) + .secretAccessKey(values.secretKey as String) + .build()) + } + } + + Region getRegion(){ + this.region + } + + AwsCredentialsProvider getAwsCredentialsProvider(){ + this.awsCredentialsProvider + } + + @Override + protected String resolveProjectName(String path){ + log.debug ("Resolving project name from $path. returning ") + if (!server.startsWith('s3://')) + new AbortOperationException("S3 project server doesn't start with s3://") + return "${server.substring('s3://'.size())}/$path" + } + +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryFactory.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryFactory.groovy new file mode 100644 index 000000000..a6321e492 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryFactory.groovy @@ -0,0 +1,85 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cloud.aws.scm.jgit.TransportS3 +import nextflow.plugin.Priority +import nextflow.scm.GitUrl +import nextflow.scm.ProviderConfig +import nextflow.scm.RepositoryFactory +import nextflow.scm.RepositoryProvider + +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Implements a factory to create an instance of {@link S3RepositoryProvider} + * + * @author Jorge Ejarque + */ +@Slf4j +@Priority(-10) +@CompileStatic +class S3RepositoryFactory extends RepositoryFactory{ + + private static AtomicBoolean registered = new AtomicBoolean(false) + + @Override + protected RepositoryProvider createProviderInstance(ProviderConfig config, String project) { + if (!registered.get()) { + registered.set(true) + TransportS3.register() + } + + return config.platform == 's3' + ? new S3RepositoryProvider(project, config) + : null + } + + @Override + protected ProviderConfig getConfig(List providers, GitUrl url) { + // do not care about non AWS codecommit url + if( url.protocol != 's3' ) + return null + + // S3 repository config depends on the bucket name stored as domain + def config = providers.find( it -> it.domain == url.domain ) + if( config ) { + log.debug "Git url=$url (1) -> config=$config" + return config + } + + // still nothing, create a new instance + config = new S3ProviderConfig(url.domain) + + + return config + } + + @Override + protected ProviderConfig createConfigInstance(String name, Map attrs) { + final copy = new HashMap(attrs) + return copy.platform == 's3' + ? new S3ProviderConfig(name, copy) + : null + } + + + + +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryProvider.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryProvider.groovy new file mode 100644 index 000000000..f5c8465fb --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/S3RepositoryProvider.groovy @@ -0,0 +1,164 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm + +import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.util.logging.Slf4j +import nextflow.cloud.aws.scm.jgit.S3GitCredentialsProvider +import nextflow.exception.AbortOperationException +import nextflow.scm.ProviderConfig +import nextflow.scm.RepositoryProvider +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.api.errors.TransportException +import org.eclipse.jgit.transport.CredentialsProvider + +import java.nio.file.Files + + +/** + * Implements a repository provider for git-remote-s3 repositories. + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class S3RepositoryProvider extends RepositoryProvider { + + S3RepositoryProvider(String project, ProviderConfig config) { + assert config instanceof S3ProviderConfig + log.debug("Creating S3 repository provider for $project") + this.project = project + this.config = config + } + /** {@inheritDoc} **/ + @Memoized + @Override + CredentialsProvider getGitCredentials() { + final providerConfig = this.config as S3ProviderConfig + final credentials = new S3GitCredentialsProvider() + if( providerConfig.region ) + credentials.setRegion(providerConfig.region) + if( providerConfig.awsCredentialsProvider ) + credentials.setAwsCredentialsProvider(providerConfig.awsCredentialsProvider) + return credentials + } + + /** {@inheritDoc} **/ + // called by AssetManager + // used to set credentials for a clone, pull, fetch, operation + @Override + boolean hasCredentials() { + // set to true + // uses AWS Credentials instead of username : password + // see getGitCredentials() + return true + } + + /** {@inheritDoc} **/ + @Override + String getName() { return project } + + /** {@inheritDoc} **/ + @Override + String getEndpointUrl() { + return "s3://$project" + } + + /** {@inheritDoc} **/ + // not used, but the abstract method needs to be overridden + @Override + String getContentUrl( String path ) { + throw new UnsupportedOperationException() + } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + String getCloneUrl() { getEndpointUrl() } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + String getRepositoryUrl() { getEndpointUrl() } + + /** {@inheritDoc} **/ + // called by AssetManager + // called by RepositoryProvider.readText() + @Override + byte[] readBytes( String path ) { + log.debug("Reading $path") + //Not possible to get a single file requires to clone the branch and get the file + final tmpDir = Files.createTempDirectory("s3-git-remote") + final command = Git.cloneRepository() + .setURI(getEndpointUrl()) + .setDirectory(tmpDir.toFile()) + .setCredentialsProvider(getGitCredentials()) + if( revision ) + command.setBranch(revision) + try { + command.call() + final file = tmpDir.resolve(path) + return file.getBytes() + } + catch (Exception e) { + log.debug(" unable to retrieve file: $path from repo: $project", e) + return null + } + finally{ + tmpDir.deleteDir() + } + } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + void validateRepo() { + // Nothing to check + } + + private String errMsg(Exception e) { + def msg = "Unable to access Git repository" + if( e.message ) + msg + " - ${e.message}" + else + msg += ": " + getCloneUrl() + return msg + } + + @Override + List getBranches() { + try { + return super.getBranches() + } + catch ( TransportException e) { + throw new AbortOperationException(errMsg(e), e) + } + } + + @Override + List getTags() { + try { + return super.getTags() + } + catch (TransportException e) { + throw new AbortOperationException(errMsg(e), e) + } + } + + +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3BaseConnection.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3BaseConnection.java new file mode 100644 index 000000000..429f07298 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3BaseConnection.java @@ -0,0 +1,186 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm.jgit; + +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.transport.Connection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Base class for connections with a git-remote-s3 compatibility + * @author Jorge Ejarque + * + */ +public class S3BaseConnection implements Connection { + + protected static final Logger log = LoggerFactory.getLogger(S3BaseConnection.class); + protected TransportS3 transport; + protected String bucket; + protected String key; + protected S3Client s3; + private final Map advertisedRefs = new HashMap(); + + public S3BaseConnection(TransportS3 transport) { + this.transport = transport; + this.bucket = transport.getURI().getHost(); + this.key = transport.getURI().getPath().substring(1); + S3GitCredentialsProvider credentials = (S3GitCredentialsProvider) transport.getCredentialsProvider(); + this.s3 = S3Client.builder() + .region(credentials.getRegion()) + .credentialsProvider(credentials.getAwsCredentialsProvider()) + .build(); + try { + loadRefsMap(); + }catch (IOException e){ + final String message = String.format("Unable to get refs from remote s3://%s/%s", bucket, key); + throw new RuntimeException(message, e); + } + log.trace("Created S3 Connection for s3://{}/{}", bucket, key); + } + + private String getDefaultBranchRef() throws IOException { + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(key + "/HEAD") + .build(); + + try (ResponseInputStream inputStream = s3.getObject(getObjectRequest)) { + return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + + + private void loadRefsMap() throws IOException { + final String defaultBranch = getDefaultBranchRef(); + addRefs("heads"); + addRefs("tags"); + Ref target = advertisedRefs.get(defaultBranch); + if (target != null) + advertisedRefs.put(Constants.HEAD, new SymbolicRef(Constants.HEAD, target)); + } + + private void addRefs(String refType) { + final List list = s3.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(key + "/refs/" + refType) + .build() + ).contents(); + + if (list == null || list.isEmpty()) { + log.debug("No {} refs found for s3://{}/{}", refType, bucket, key); + return; + } + + for (S3Object obj : list) { + addRef(obj); + } + } + private void addRef(S3Object obj) { + String key = obj.key(); + BranchData branch = BranchData.fromKey(key); + String type = branch.getType(); + if ("heads".equals(type)) { + advertisedRefs.put(branch.getRefName(), new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, branch.getRefName(), branch.getObjectId())); + } else if ("tags".equals(type)) { + advertisedRefs.put(branch.getRefName(), new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, branch.getRefName(), branch.getObjectId())); + } + } + + @Override + public Map getRefsMap() { + return advertisedRefs; + } + + @Override + public Collection getRefs() { + return advertisedRefs.values(); + } + + @Override + public Ref getRef(String name) { + return advertisedRefs.get(name); + } + + @Override + public void close() { } + + @Override + public String getMessages() { + return ""; + } + + @Override + public String getPeerUserAgent() { + return ""; + } + + static class BranchData{ + private String type; + private String simpleName; + private String refName; + private ObjectId objectId; + + private BranchData(String type, String simpleName, ObjectId objectId){ + this.type = type; + this.simpleName = simpleName; + this.refName = String.format("refs/%s/%s" , type, simpleName); + this.objectId = objectId; + } + public static BranchData fromKey(String key){ + String[] parts = key.split("/"); + if (parts.length < 5) throw new RuntimeException("Incorrect key parts"); + // Expect: repo-path/refs///.bundle + final String type = parts[parts.length - 3]; + final String rBranch = parts[parts.length - 2]; + final String sha = parts[parts.length - 1].replace(".bundle", ""); + return new BranchData(type, rBranch, ObjectId.fromString(sha)); + + } + + public String getType() { + return type; + } + + public String getSimpleName() { + return simpleName; + } + + public String getRefName() { + return refName; + } + + public ObjectId getObjectId() { + return objectId; + } + } + +} \ No newline at end of file diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3FetchConnection.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3FetchConnection.java new file mode 100644 index 000000000..7120bb3b0 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3FetchConnection.java @@ -0,0 +1,137 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.transport.*; +import org.eclipse.jgit.util.FileUtils; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Fetch Connection implementation compatible with git-remote-s3 storage. + * + * @author Jorge Ejarque + */ +public class S3FetchConnection extends S3BaseConnection implements FetchConnection { + + public S3FetchConnection(TransportS3 transport) { + super(transport); + + } + + @Override + public void fetch(ProgressMonitor monitor, Collection want, Set have) throws TransportException { + Path tmpdir = null; + try { + tmpdir = Files.createTempDirectory("s3-remote-git-"); + for (Ref r : want) { + downloadBundle(r, tmpdir, monitor); + } + }catch (IOException e){ + throw new TransportException(transport.getURI(), "Exception fetching branches", e); + }finally { + if (tmpdir != null) + try { + FileUtils.delete(tmpdir.toFile(), FileUtils.RECURSIVE); + }catch (IOException e){ + throw new TransportException(transport.getURI(), "Exception fetching branches", e); + } + } + + } + + private void downloadBundle(Ref r, Path tmpdir, ProgressMonitor monitor) throws IOException{ + log.debug("Fetching {} in {}", r.getName(), tmpdir); + final List list = s3.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(key + '/' + r.getName() + '/') + .build() + ).contents(); + + if( list == null || list.isEmpty() ) { + throw new TransportException(transport.getURI(), "No bundle for " + r.getName()); + } + + if( list.size() > 1 ){ + throw new TransportException(transport.getURI(), " More than one bundle for " +r.getName()); + } + String key = list.get(0).key(); + String bundleName = key.substring(key.lastIndexOf('/') + 1); + Path localBundle = tmpdir.resolve(bundleName); + Files.createDirectories(localBundle.getParent()); + log.trace("Downloading bundle {} for branch {} in {} ", key, r.getName(), localBundle); + + s3.getObject( + GetObjectRequest.builder().bucket(bucket).key(key).build(), + localBundle + ); + parseBundle( localBundle, monitor); + + } + + private void parseBundle( Path localBundle, ProgressMonitor monitor) throws TransportException { + try { + List specs = new ArrayList<>(); + specs.add(new RefSpec().setForceUpdate(true).setSourceDestination(Constants.R_REFS + '*', Constants.R_REFS + '*')); + Transport.open( transport.getLocal(), new URIish( localBundle.toUri().toString() ) ).fetch(monitor, specs); + + } catch (IOException | RuntimeException | URISyntaxException err) { + close(); + throw new TransportException(transport.getURI(), err.getMessage(), err); + } + } + + @Override + public void fetch(ProgressMonitor monitor, Collection want, Set have, OutputStream out) throws TransportException { + fetch(monitor,want,have); + } + + @Override + public boolean didFetchIncludeTags() { + return false; + } + + @Override + public boolean didFetchTestConnectivity() { + return false; + } + + @Override + public void setPackLockMessage(String message) { + // No pack lock message supported. + } + + @Override + public Collection getPackLocks() { + return Collections.emptyList(); + } + + +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3GitCredentialsProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3GitCredentialsProvider.java new file mode 100644 index 000000000..eb88fd0cd --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3GitCredentialsProvider.java @@ -0,0 +1,66 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm.jgit; + +import org.eclipse.jgit.errors.UnsupportedCredentialItem; +import org.eclipse.jgit.transport.CredentialItem; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +/** + * JGit credentials provider wrapper for the AWS credentialsProvider and other client configuration parameters. + * + * @author Jorge Ejarque + */ +public class S3GitCredentialsProvider extends CredentialsProvider { + + private Region region; + private AwsCredentialsProvider awsCredentialsProvider; + + public void setAwsCredentialsProvider(AwsCredentialsProvider provider){ + this.awsCredentialsProvider = provider; + } + public void setRegion(Region region) { + this.region = region; + } + + public Region getRegion(){ + return region != null ? region : Region.US_EAST_1; + } + + public AwsCredentialsProvider getAwsCredentialsProvider(){ + return awsCredentialsProvider != null ? awsCredentialsProvider : DefaultCredentialsProvider.builder().build(); + } + + @Override + public boolean isInteractive() { + return false; + } + + @Override + public boolean supports(CredentialItem... items) { + return false; + } + + @Override + public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem { + return false; + } +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3PushConnection.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3PushConnection.java new file mode 100644 index 000000000..49aa4dbc6 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/S3PushConnection.java @@ -0,0 +1,173 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.BundleWriter; +import org.eclipse.jgit.transport.PushConnection; +import org.eclipse.jgit.transport.RemoteRefUpdate; +import org.eclipse.jgit.util.FileUtils; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Push connection implementation compatible with git-remote-s3 storage. + * + * @author Jorge Ejarque + */ +public class S3PushConnection extends S3BaseConnection implements PushConnection { + + public S3PushConnection(TransportS3 transport) { + super(transport); + } + + @Override + public void push(ProgressMonitor monitor, Map refUpdates) throws TransportException { + Path tmpdir = null; + try { + tmpdir = Files.createTempDirectory("s3-remote-git-"); + for (Map.Entry entry : refUpdates.entrySet()) { + pushBranch(entry, tmpdir); + } + }catch (IOException e){ + throw new TransportException(transport.getURI(), "Exception fetching branches", e); + }finally { + if (tmpdir != null) + try { + FileUtils.delete(tmpdir.toFile(), FileUtils.RECURSIVE); + }catch (IOException e){ + throw new TransportException(transport.getURI(), "Exception fetching branches", e); + } + } + + } + + private void pushBranch(Map.Entry entry, Path tmpdir) throws IOException { + final Ref ref = transport.getLocal().findRef(entry.getKey()); + if( ref == null || ref.getObjectId() == null) { + throw new IllegalStateException("Branch ${branch} not found"); + } + S3Object oldObject = checkExistingObjectInBranch(entry.getKey()); + if( oldObject != null && isSameObjectId(oldObject, ref.getObjectId())){ + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.UP_TO_DATE); + return; + } + if( oldObject != null && !isCommitInBranch(oldObject, ref)) { + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.REJECTED_REMOTE_CHANGED); + return; + } + log.trace("Generating bundle for branch {} in {}", entry.getKey(), tmpdir); + Path bundleFile = bundle(ref, tmpdir); + String objectKey = String.format("%s/%s/%s", key, entry.getKey(), bundleFile.getFileName().toString()); + + log.trace("Uploading bundle {} to s3://{}/{}", bundleFile, bucket, objectKey); + s3.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(), + bundleFile); + if( oldObject != null ){ + log.trace("Deleting old bundle s3://{}/{}",bucket,oldObject.key()); + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(bucket) + .key(oldObject.key()) + .build() + ); + } + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.OK); + } + + private boolean isSameObjectId(S3Object s3object, ObjectId commitId){ + return BranchData.fromKey(s3object.key()).getObjectId().name().equals(commitId.name()); + } + + private void setUpdateStatus(RemoteRefUpdate update, RemoteRefUpdate.Status status) { + try { + Field statusField = RemoteRefUpdate.class.getDeclaredField("status"); + statusField.setAccessible(true); + statusField.set(update, status); + } catch (Exception e) { + throw new RuntimeException("Unable to set status on RemoteRefUpdate", e); + } + } + + public boolean isCommitInBranch(S3Object s3Object, Ref branchRef) throws IOException { + ObjectId commitId = BranchData.fromKey(s3Object.key()).getObjectId(); + try (RevWalk walk = new RevWalk(transport.getLocal())) { + RevCommit branchTip = walk.parseCommit(branchRef.getObjectId()); + RevCommit targetCommit = walk.parseCommit(commitId); + + // Check if the commit is reachable from the branch tip + return walk.isMergedInto(targetCommit, branchTip); + } + } + + private S3Object checkExistingObjectInBranch(String name) throws TransportException { + final List list = s3.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(key + '/' + name + '/') + .build() + ).contents(); + + if( list == null || list.isEmpty() ) { + return null; + } + + if( list.size() > 1 ){ + throw new TransportException(transport.getURI(), " More than one bundle for " + name); + } + return list.get(0); + } + + @Override + public void push(ProgressMonitor monitor, Map refUpdates, OutputStream out) throws TransportException { + push(monitor, refUpdates); + + } + + private Path bundle(Ref ref, Path tmpdir) throws IOException { + final BundleWriter writer = new BundleWriter(transport.getLocal()); + Path bundleFile = tmpdir.resolve(ref.getObjectId() +".bundle"); + writer.include(ref); + try (OutputStream out = new FileOutputStream(bundleFile.toFile())) { + writer.writeBundle(NullProgressMonitor.INSTANCE, out); + } + return bundleFile; + } + + @Override + public void close() { + + } +} \ No newline at end of file diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/TransportS3.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/TransportS3.java new file mode 100644 index 000000000..4e3938139 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/scm/jgit/TransportS3.java @@ -0,0 +1,89 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.aws.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.*; + +import java.util.Collections; +import java.util.Set; + +/** + * JGit transport implementation compatible with git-remote-s3 storage. + * + * @author Jorge Ejarque + */ +public class TransportS3 extends Transport { + + public static final TransportProtocol PROTO_S3 = new S3TransportProtocol(); + + public TransportS3(Repository local, URIish uri) throws TransportException { + super(local, uri); + } + + @Override + public FetchConnection openFetch() throws TransportException { + return new S3FetchConnection(this); + } + + @Override + public PushConnection openPush() throws TransportException { + return new S3PushConnection(this); + } + + // Optional: Clean up if needed + @Override + public void close() { + // cleanup resources if needed + } + + public Repository getLocal(){ + return this.local; + } + + public static class S3TransportProtocol extends TransportProtocol { + @Override + public String getName() { + return "Amazon S3"; + } + + @Override + public Set getSchemes() { + return Collections.singleton("s3"); + } + + @Override + public boolean canHandle(URIish uri, Repository local, String remoteName) { + return "s3".equals(uri.getScheme()); + } + + @Override + public Transport open(URIish uri, Repository local, String remoteName) throws TransportException { + try { + return new TransportS3(local, uri); + } catch (TransportException e) { + throw e; + } + } + } + + public static void register() { + Transport.register(PROTO_S3); + } +} + diff --git a/plugins/nf-amazon/src/resources/META-INF/extensions.idx b/plugins/nf-amazon/src/resources/META-INF/extensions.idx index 50517c11c..cff26a8e3 100644 --- a/plugins/nf-amazon/src/resources/META-INF/extensions.idx +++ b/plugins/nf-amazon/src/resources/META-INF/extensions.idx @@ -19,3 +19,4 @@ nextflow.cloud.aws.util.S3PathSerializer nextflow.cloud.aws.util.S3PathFactory nextflow.cloud.aws.fusion.AwsFusionEnv nextflow.cloud.aws.mail.AwsMailProvider +nextflow.cloud.aws.scm.S3RepositoryFactory