From 500e1e0c36ecd252758dc5db0c1c1564b814becb Mon Sep 17 00:00:00 2001 From: Nico Duldhardt Date: Sun, 12 Oct 2025 12:01:55 +0200 Subject: [PATCH 1/2] add atomic conditional puts for azureblob-sdk --- .gitignore | 5 ++ README.md | 2 +- .../java/org/gaul/s3proxy/PutOptions2.java | 80 +++++++++++++++++++ .../java/org/gaul/s3proxy/S3ProxyHandler.java | 30 ++++--- .../s3proxy/azureblob/AzureBlobStore.java | 25 ++++++ 5 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/gaul/s3proxy/PutOptions2.java diff --git a/.gitignore b/.gitignore index 8fce8334..7c0d9aea 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,8 @@ s3proxy.iml # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* target/ + +# files created during tests +__blobstorage__/ +AzuriteConfig +__azurite_db* diff --git a/README.md b/README.md index 8ead9c9f..a4411b1d 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ S3Proxy has broad compatibility with the S3 API, however, it does not support: S3Proxy emulates the following operations: -* conditional PUT object when using If-Match or If-None-Match +* conditional PUT object when using If-Match or If-None-Match, unless the `azureblob-sdk` provider is used * copy multi-part objects, see [#76](https://github.com/gaul/s3proxy/issues/76) S3Proxy has basic CORS preflight and actual request/response handling. It can be configured within the properties diff --git a/src/main/java/org/gaul/s3proxy/PutOptions2.java b/src/main/java/org/gaul/s3proxy/PutOptions2.java new file mode 100644 index 00000000..c6828fe5 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/PutOptions2.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014-2025 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import javax.annotation.Nullable; + +import org.jclouds.blobstore.options.PutOptions; + +/** + * This class extends PutOptions to support conditional put operations via + * the If-Match and If-None-Match headers. + */ +public final class PutOptions2 extends PutOptions { + @Nullable + private String ifMatch; + @Nullable + private String ifNoneMatch; + + public PutOptions2() { + super(); + } + + public PutOptions2(PutOptions options) { + super(options.isMultipart(), options.getUseCustomExecutor(), + options.getCustomExecutor()); + this.setBlobAccess(options.getBlobAccess()); + + if (options instanceof PutOptions2) { + PutOptions2 other = (PutOptions2) options; + this.ifMatch = other.ifMatch; + this.ifNoneMatch = other.ifNoneMatch; + } + } + + @Nullable + public String getIfMatch() { + return ifMatch; + } + + public PutOptions2 ifMatch(@Nullable String etag) { + this.ifMatch = etag; + return this; + } + + @Nullable + public String getIfNoneMatch() { + return ifNoneMatch; + } + + public PutOptions2 ifNoneMatch(@Nullable String etag) { + this.ifNoneMatch = etag; + return this; + } + + public boolean hasConditionalHeaders() { + return ifMatch != null || ifNoneMatch != null; + } + + @Override + public String toString() { + String s = super.toString(); + return s.substring(0, s.length() - 1) + + ", ifMatch=" + ifMatch + + ", ifNoneMatch=" + ifNoneMatch + "]"; + } +} diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index fcfdcd16..87cd68df 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -1994,15 +1994,26 @@ private void handlePutBlob(HttpServletRequest request, throw new S3Exception(S3ErrorCode.ENTITY_TOO_LARGE); } - // Handle If-Match and If-None-Match headers for PUT operations. - // Unlike GET operations which use GetOptions to pass these conditions - // to jclouds, PUT operations lack a PutOptions equivalent in jclouds. - // Therefore, we manually fetch the blob metadata and validate ETags. - // TODO: this is an emulated operation and therefore not atomic. String ifMatch = request.getHeader(HttpHeaders.IF_MATCH); String ifNoneMatch = request.getHeader(HttpHeaders.IF_NONE_MATCH); + String blobStoreType = getBlobStoreType(blobStore); + + // Azure only supports If-None-Match: *, not If-Match: * + // Handle If-Match: * manually for the azureblob-sdk provider. + // Note: this is a non-atomic operation (HEAD then PUT). + if (ifMatch != null && ifMatch.equals("*") && + blobStoreType.equals("azureblob-sdk")) { + BlobMetadata metadata = blobStore.blobMetadata(containerName, blobName); + if (metadata == null) { + throw new S3Exception(S3ErrorCode.PRECONDITION_FAILED); + } + ifMatch = null; + } - if (ifMatch != null || ifNoneMatch != null) { + // Emulate conditional put for backends without native support. + // Note: this is a non-atomic operation (HEAD then PUT). + if ((ifMatch != null || ifNoneMatch != null) && + !blobStoreType.equals("azureblob-sdk")) { BlobMetadata metadata = blobStore.blobMetadata(containerName, blobName); if (ifMatch != null) { if (ifMatch.equals("*")) { @@ -2055,9 +2066,10 @@ private void handlePutBlob(HttpServletRequest request, return; } - var options = new PutOptions().setBlobAccess(access); - - String blobStoreType = getBlobStoreType(blobStore); + var options = new PutOptions2(); + options.setBlobAccess(access); + options.ifMatch(ifMatch); + options.ifNoneMatch(ifNoneMatch); if (blobStoreType.equals("azureblob") && contentLength > 256 * 1024 * 1024) { options.multipart(true); diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java index af522b4f..4ce81d4e 100644 --- a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java @@ -66,6 +66,7 @@ import jakarta.inject.Singleton; import jakarta.ws.rs.core.Response.Status; +import org.gaul.s3proxy.PutOptions2; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.KeyNotFoundException; @@ -395,6 +396,20 @@ public String putBlob(String container, Blob blob, PutOptions options) { blob.getMetadata().getTier())); } + if (options instanceof PutOptions2) { + var putOptions2 = (PutOptions2) options; + if (putOptions2.hasConditionalHeaders()) { + var conditions = new BlobRequestConditions(); + if (putOptions2.getIfMatch() != null) { + conditions.setIfMatch(putOptions2.getIfMatch()); + } + if (putOptions2.getIfNoneMatch() != null) { + conditions.setIfNoneMatch(putOptions2.getIfNoneMatch()); + } + azureOptions.setRequestConditions(conditions); + } + } + try (var os = client.getBlobOutputStream( azureOptions, /*context=*/ null)) { is.transferTo(os); @@ -749,6 +764,16 @@ private void translateAndRethrowException(BlobStorageException bse, .build(); throw new HttpResponseException( new HttpCommand(request), response, bse); + } else if (code.equals(BlobErrorCode.BLOB_ALREADY_EXISTS)) { + var request = HttpRequest.builder() + .method("PUT") + .endpoint(endpoint) + .build(); + var response = HttpResponse.builder() + .statusCode(Status.PRECONDITION_FAILED.getStatusCode()) + .build(); + throw new HttpResponseException( + new HttpCommand(request), response, bse); } else if (code.equals(BlobErrorCode.INVALID_OPERATION)) { var request = HttpRequest.builder() .method("GET") From 544467ef0d8cd668cec09b7b2160ff7e9074fc03 Mon Sep 17 00:00:00 2001 From: Nico Duldhardt Date: Sun, 12 Oct 2025 19:00:24 +0200 Subject: [PATCH 2/2] refactor PutOptions2 interface --- .../java/org/gaul/s3proxy/PutOptions2.java | 39 ++++++++++++++++--- .../java/org/gaul/s3proxy/S3ProxyHandler.java | 8 ++-- .../s3proxy/azureblob/AzureBlobStore.java | 15 +++---- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/PutOptions2.java b/src/main/java/org/gaul/s3proxy/PutOptions2.java index c6828fe5..3467b5f9 100644 --- a/src/main/java/org/gaul/s3proxy/PutOptions2.java +++ b/src/main/java/org/gaul/s3proxy/PutOptions2.java @@ -18,10 +18,13 @@ import javax.annotation.Nullable; +import com.google.common.util.concurrent.ListeningExecutorService; + +import org.jclouds.blobstore.domain.BlobAccess; import org.jclouds.blobstore.options.PutOptions; /** - * This class extends PutOptions to support conditional put operations via + * This class extends jclouds' PutOptions to support conditional put operations via * the If-Match and If-None-Match headers. */ public final class PutOptions2 extends PutOptions { @@ -51,7 +54,7 @@ public String getIfMatch() { return ifMatch; } - public PutOptions2 ifMatch(@Nullable String etag) { + public PutOptions2 setIfMatch(@Nullable String etag) { this.ifMatch = etag; return this; } @@ -61,13 +64,39 @@ public String getIfNoneMatch() { return ifNoneMatch; } - public PutOptions2 ifNoneMatch(@Nullable String etag) { + public PutOptions2 setIfNoneMatch(@Nullable String etag) { this.ifNoneMatch = etag; return this; } - public boolean hasConditionalHeaders() { - return ifMatch != null || ifNoneMatch != null; + @Override + public PutOptions2 setBlobAccess(BlobAccess blobAccess) { + super.setBlobAccess(blobAccess); + return this; + } + + @Override + public PutOptions2 multipart() { + super.multipart(); + return this; + } + + @Override + public PutOptions2 multipart(boolean val) { + super.multipart(val); + return this; + } + + @Override + public PutOptions2 multipart(ListeningExecutorService customExecutor) { + super.multipart(customExecutor); + return this; + } + + @Override + public PutOptions2 setCustomExecutor(ListeningExecutorService customExecutor) { + super.setCustomExecutor(customExecutor); + return this; } @Override diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 87cd68df..4e05c246 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -2066,10 +2066,10 @@ private void handlePutBlob(HttpServletRequest request, return; } - var options = new PutOptions2(); - options.setBlobAccess(access); - options.ifMatch(ifMatch); - options.ifNoneMatch(ifNoneMatch); + var options = new PutOptions2() + .setBlobAccess(access) + .setIfMatch(ifMatch) + .setIfNoneMatch(ifNoneMatch); if (blobStoreType.equals("azureblob") && contentLength > 256 * 1024 * 1024) { options.multipart(true); diff --git a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java index 4ce81d4e..0278cc68 100644 --- a/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java +++ b/src/main/java/org/gaul/s3proxy/azureblob/AzureBlobStore.java @@ -398,15 +398,12 @@ public String putBlob(String container, Blob blob, PutOptions options) { if (options instanceof PutOptions2) { var putOptions2 = (PutOptions2) options; - if (putOptions2.hasConditionalHeaders()) { - var conditions = new BlobRequestConditions(); - if (putOptions2.getIfMatch() != null) { - conditions.setIfMatch(putOptions2.getIfMatch()); - } - if (putOptions2.getIfNoneMatch() != null) { - conditions.setIfNoneMatch(putOptions2.getIfNoneMatch()); - } - azureOptions.setRequestConditions(conditions); + String ifMatch = putOptions2.getIfMatch(); + String ifNoneMatch = putOptions2.getIfNoneMatch(); + if (ifMatch != null || ifNoneMatch != null) { + azureOptions.setRequestConditions(new BlobRequestConditions() + .setIfMatch(ifMatch) + .setIfNoneMatch(ifNoneMatch)); } }