From 88c0ed1faed138122ec71b74bcc2201c440ec48d Mon Sep 17 00:00:00 2001 From: dlarge Date: Thu, 21 Nov 2024 21:17:21 +0000 Subject: [PATCH] Client max chunk size is now configurable and defaults to 64KB --- README.md | 3 ++- build.gradle | 4 ++-- .../storage/driver/coop/netty/http/HttpStorageDriver.java | 1 - .../storage/driver/coop/netty/http/HttpStorageDriverBase.java | 4 +++- src/main/resources/config-schema-storage-net-http.yaml | 3 +++ src/main/resources/config/defaults-storage-net-http.yaml | 3 +++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eb5f3e6..f654b6f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ |:-----------------------------------------------|:-------------|:-----------------|:-------------------------------------------------| | storage-net-http-headers | Map | { "Connection" : "keep-alive", "Date": "#{date:formatNowRfc1123()}%{date:formatNowRfc1123()}", "User-Agent" : "mongoose-storage-driver-http/4.2.6" } | Custom HTTP headers section. A user may place here a key-value pair which will be used as HTTP header. The headers will be appended to every HTTP request issued. | storage-net-http-uri-args | Map | {} | Custom URI query arguments according [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt).The headers will be appended to every HTTP request issued. -| storage-net-http-read-metadata-only | Map | false | specifies whether Mongoose issues GET request or HEAD. HEAD is used when enabled. +| storage-net-http-read-metadata-only | Map | false | Specifies whether Mongoose issues GET request or HEAD. HEAD is used when enabled. +| storage-net-http-max-chunk-size | Integer | 65536 | The limit, in bytes, at which Netty will send a chunk down the pipeline. ## 2. Custom HTTP Headers diff --git a/build.gradle b/build.gradle index a519062..f75de4a 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ repositories { description = "Mongoose is a high-load storage performance testing tool" group = "com.github.emc-mongoose" -version = "4.2.20" +version = "4.2.21" sourceCompatibility = 11 targetCompatibility = 11 @@ -41,7 +41,7 @@ ext { mongooseBase : "4.3.3", mongooseStorageDriverCoop: "4.2.21", mongooseStorageDriverNetty: "4.2.18", - netty : "4.1.25.Final", + netty : "4.1.115.Final", scala : "2.12.6", slf4j : "1.7.25", ] diff --git a/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriver.java b/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriver.java index 14b1224..475f940 100644 --- a/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriver.java +++ b/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriver.java @@ -13,7 +13,6 @@ public interface HttpStorageDriver> int REQ_LINE_LEN = 1024; int HEADERS_LEN = 2048; - int CHUNK_SIZE = 8192; String KEY_CONTENT = "content"; } diff --git a/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriverBase.java b/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriverBase.java index ade584a..98b4878 100644 --- a/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriverBase.java +++ b/src/main/java/com/emc/mongoose/storage/driver/coop/netty/http/HttpStorageDriverBase.java @@ -86,6 +86,7 @@ public abstract class HttpStorageDriverBase uriQueryInput; protected final ChannelFutureListener httpReqSentCallback = this::sendHttpRequestComplete; protected final boolean readMetadataOnly; + protected final int maxChunkSize; protected HttpStorageDriverBase( final String testStepId, @@ -115,6 +116,7 @@ protected HttpStorageDriverBase( if (readMetadataOnly) { Loggers.MSG.info("Reading metadata only (HEAD requests)"); } + this.maxChunkSize = httpConfig.intVal("max-chunk-size"); final var uriArgs = httpConfig. mapVal("uri-args"); final var uriQueryExpr = uriArgs.entrySet().stream() .map(entry -> entry.getKey() + '=' + entry.getValue()) @@ -178,7 +180,7 @@ protected void appendHandlers(final Channel channel) { super.appendHandlers(channel); channel .pipeline() - .addLast(new HttpClientCodec(REQ_LINE_LEN, HEADERS_LEN, CHUNK_SIZE, true)) + .addLast(new HttpClientCodec(REQ_LINE_LEN, HEADERS_LEN, maxChunkSize, true)) .addLast(new ChunkedWriteHandler()); } diff --git a/src/main/resources/config-schema-storage-net-http.yaml b/src/main/resources/config-schema-storage-net-http.yaml index c5f1a58..84123d6 100644 --- a/src/main/resources/config-schema-storage-net-http.yaml +++ b/src/main/resources/config-schema-storage-net-http.yaml @@ -9,3 +9,6 @@ storage: only: boolean uri: args: map + max: + chunk: + size: int diff --git a/src/main/resources/config/defaults-storage-net-http.yaml b/src/main/resources/config/defaults-storage-net-http.yaml index 23723dc..53eeffd 100644 --- a/src/main/resources/config/defaults-storage-net-http.yaml +++ b/src/main/resources/config/defaults-storage-net-http.yaml @@ -13,3 +13,6 @@ storage: only: false uri: args: {} + max: + chunk: + size: 65536