diff --git a/.gitignore b/.gitignore index 8fce8334..7c0d9aea 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,8 @@ s3proxy.iml # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* target/ + +# files created during tests +__blobstorage__/ +AzuriteConfig +__azurite_db* diff --git a/README.md b/README.md index 8ead9c9f..a4411b1d 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ S3Proxy has broad compatibility with the S3 API, however, it does not support: S3Proxy emulates the following operations: -* conditional PUT object when using If-Match or If-None-Match +* conditional PUT object when using If-Match or If-None-Match, unless the `azureblob-sdk` provider is used * copy multi-part objects, see [#76](https://github.com/gaul/s3proxy/issues/76) S3Proxy has basic CORS preflight and actual request/response handling. It can be configured within the properties diff --git a/src/main/java/org/gaul/s3proxy/PutOptions2.java b/src/main/java/org/gaul/s3proxy/PutOptions2.java new file mode 100644 index 00000000..3467b5f9 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/PutOptions2.java @@ -0,0 +1,109 @@ +/* + * Copyright 2014-2025 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import javax.annotation.Nullable; + +import com.google.common.util.concurrent.ListeningExecutorService; + +import org.jclouds.blobstore.domain.BlobAccess; +import org.jclouds.blobstore.options.PutOptions; + +/** + * This class extends jclouds' PutOptions to support conditional put operations via + * the If-Match and If-None-Match headers. + */ +public final class PutOptions2 extends PutOptions { + @Nullable + private String ifMatch; + @Nullable + private String ifNoneMatch; + + public PutOptions2() { + super(); + } + + public PutOptions2(PutOptions options) { + super(options.isMultipart(), options.getUseCustomExecutor(), + options.getCustomExecutor()); + this.setBlobAccess(options.getBlobAccess()); + + if (options instanceof PutOptions2) { + PutOptions2 other = (PutOptions2) options; + this.ifMatch = other.ifMatch; + this.ifNoneMatch = other.ifNoneMatch; + } + } + + @Nullable + public String getIfMatch() { + return ifMatch; + } + + public PutOptions2 setIfMatch(@Nullable String etag) { + this.ifMatch = etag; + return this; + } + + @Nullable + public String getIfNoneMatch() { + return ifNoneMatch; + } + + public PutOptions2 setIfNoneMatch(@Nullable String etag) { + this.ifNoneMatch = etag; + return this; + } + + @Override + public PutOptions2 setBlobAccess(BlobAccess blobAccess) { + super.setBlobAccess(blobAccess); + return this; + } + + @Override + public PutOptions2 multipart() { + super.multipart(); + return this; + } + + @Override + public PutOptions2 multipart(boolean val) { + super.multipart(val); + return this; + } + + @Override + public PutOptions2 multipart(ListeningExecutorService customExecutor) { + super.multipart(customExecutor); + return this; + } + + @Override + public PutOptions2 setCustomExecutor(ListeningExecutorService customExecutor) { + super.setCustomExecutor(customExecutor); + return this; + } + + @Override + public String toString() { + String s = super.toString(); + return s.substring(0, s.length() - 1) + + ", ifMatch=" + ifMatch + + ", ifNoneMatch=" + ifNoneMatch + "]"; + } +} diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index fcfdcd16..4e05c246 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -1994,15 +1994,26 @@ private void handlePutBlob(HttpServletRequest request, throw new S3Exception(S3ErrorCode.ENTITY_TOO_LARGE); } - // Handle If-Match and If-None-Match headers for PUT operations. - // Unlike GET operations which use GetOptions to pass these conditions - // to jclouds, PUT operations lack a PutOptions equivalent in jclouds. - // Therefore, we manually fetch the blob metadata and validate ETags. - // TODO: this is an emulated operation and therefore not atomic. String ifMatch = request.getHeader(HttpHeaders.IF_MATCH); String ifNoneMatch = request.getHeader(HttpHeaders.IF_NONE_MATCH); + String blobStoreType = getBlobStoreType(blobStore); + + // Azure only supports If-None-Match: *, not If-Match: * + // Handle If-Match: * manually for the azureblob-sdk provider. + // Note: this is a non-atomic operation (HEAD then PUT). + if (ifMatch != null && ifMatch.equals("*") && + blobStoreType.equals("azureblob-sdk")) { + BlobMetadata metadata = blobStore.blobMetadata(containerName, blobName); + if (metadata == null) { + throw new S3Exception(S3ErrorCode.PRECONDITION_FAILED); + } + ifMatch = null; + } - if (ifMatch != null || ifNoneMatch != null) { + // Emulate conditional put for backends without native support. + // Note: this is a non-atomic operation (HEAD then PUT). + if ((ifMatch != null || ifNoneMatch != null) && + !blobStoreType.equals("azureblob-sdk")) { BlobMetadata metadata = blobStore.blobMetadata(containerName, blobName); if (ifMatch != null) { if (ifMatch.equals("*")) { @@ -2055,9 +2066,10 @@ private void handlePutBlob(HttpServletRequest request, return; } - var options = new PutOptions().setBlobAccess(access); - - String blobStoreType = getBlobStoreType(blobStore); + var options = new PutOptions2() + .setBlobAccess(access) + .setIfMatch(ifMatch) + .setIfNoneMatch(ifNoneMatch); if (blobStoreType.equals("azureblob") && contentLength > 256 * 1024 * 1024) { options.multipart(true); diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java index af522b4f..0278cc68 100644 --- a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java @@ -66,6 +66,7 @@ import jakarta.inject.Singleton; import jakarta.ws.rs.core.Response.Status; +import org.gaul.s3proxy.PutOptions2; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.KeyNotFoundException; @@ -395,6 +396,17 @@ public String putBlob(String container, Blob blob, PutOptions options) { blob.getMetadata().getTier())); } + if (options instanceof PutOptions2) { + var putOptions2 = (PutOptions2) options; + String ifMatch = putOptions2.getIfMatch(); + String ifNoneMatch = putOptions2.getIfNoneMatch(); + if (ifMatch != null || ifNoneMatch != null) { + azureOptions.setRequestConditions(new BlobRequestConditions() + .setIfMatch(ifMatch) + .setIfNoneMatch(ifNoneMatch)); + } + } + try (var os = client.getBlobOutputStream( azureOptions, /*context=*/ null)) { is.transferTo(os); @@ -749,6 +761,16 @@ private void translateAndRethrowException(BlobStorageException bse, .build(); throw new HttpResponseException( new HttpCommand(request), response, bse); + } else if (code.equals(BlobErrorCode.BLOB_ALREADY_EXISTS)) { + var request = HttpRequest.builder() + .method("PUT") + .endpoint(endpoint) + .build(); + var response = HttpResponse.builder() + .statusCode(Status.PRECONDITION_FAILED.getStatusCode()) + .build(); + throw new HttpResponseException( + new HttpCommand(request), response, bse); } else if (code.equals(BlobErrorCode.INVALID_OPERATION)) { var request = HttpRequest.builder() .method("GET")