From 7879aa222a0ffa5dccd5cf25f90d4b81349c658c Mon Sep 17 00:00:00 2001 From: Will Vuong Date: Mon, 9 Sep 2024 11:46:33 -0400 Subject: [PATCH 1/3] Allow configuring Netty compression content size threshold --- vertx-core/src/main/asciidoc/http.adoc | 8 ++++++-- .../core/http/HttpServerOptionsConverter.java | 6 ++++++ .../io/vertx/core/http/HttpServerOptions.java | 17 +++++++++++++++++ .../impl/HttpServerConnectionInitializer.java | 6 +++--- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/vertx-core/src/main/asciidoc/http.adoc b/vertx-core/src/main/asciidoc/http.adoc index ec7845a255c..c4523982a89 100644 --- a/vertx-core/src/main/asciidoc/http.adoc +++ b/vertx-core/src/main/asciidoc/http.adoc @@ -756,9 +756,9 @@ Whenever the response needs to be sent without compression you can set the heade Be aware that compression may be able to reduce network traffic but is more CPU-intensive. -To address this latter issue Vert.x allows you to tune the 'compression level' parameter that is native of the gzip/deflate compression algorithms. +To address this latter issue Vert.x allows you to tune the 'compression level' parameter that is native of the gzip/deflate compression algorithms and also set the minimum response content size threshold for compression. -Compression level allows to configure gizp/deflate algorithms in terms of the compression ratio of the resulting data and the computational cost of the compress/decompress operation. +Compression level allows to configure gzip/deflate algorithms in terms of the compression ratio of the resulting data and the computational cost of the compress/decompress operation. The compression level is an integer value ranged from '1' to '9', where '1' means lower compression ratio but fastest algorithm and '9' means maximum compression ratio available but a slower algorithm. @@ -771,6 +771,10 @@ the more the level increases. By default - if compression is enabled via {@link io.vertx.core.http.HttpServerOptions#setCompressionSupported} - Vert.x will use '6' as compression level, but the parameter can be configured to address any case with {@link io.vertx.core.http.HttpServerOptions#setCompressionLevel}. +It may not make sense to compress responses under certain size thresholds where the trade-off between CPU and saved network bytes is not beneficial. +The minimum response content size threshold for compression can be configured via {@link io.vertx.core.http.HttpServerOptions#setCompressionContentSizeThreshold}. +For example, if set to '100', responses under 100 bytes will not be compressed. By default, it is '0' which means all content can be compressed. + === HTTP compression algorithms Vert.x supports out of the box deflate and gzip. diff --git a/vertx-core/src/main/generated/io/vertx/core/http/HttpServerOptionsConverter.java b/vertx-core/src/main/generated/io/vertx/core/http/HttpServerOptionsConverter.java index 8854c2d20da..22a0662c0df 100644 --- a/vertx-core/src/main/generated/io/vertx/core/http/HttpServerOptionsConverter.java +++ b/vertx-core/src/main/generated/io/vertx/core/http/HttpServerOptionsConverter.java @@ -28,6 +28,11 @@ static void fromJson(Iterable> json, HttpSer obj.setCompressionLevel(((Number)member.getValue()).intValue()); } break; + case "compressionContentSizeThreshold": + if (member.getValue() instanceof Number) { + obj.setCompressionContentSizeThreshold(((Number)member.getValue()).intValue()); + } + break; case "acceptUnmaskedFrames": if (member.getValue() instanceof Boolean) { obj.setAcceptUnmaskedFrames((Boolean)member.getValue()); @@ -189,6 +194,7 @@ static void toJson(HttpServerOptions obj, JsonObject json) { static void toJson(HttpServerOptions obj, java.util.Map json) { json.put("compressionSupported", obj.isCompressionSupported()); json.put("compressionLevel", obj.getCompressionLevel()); + json.put("compressionContentSizeThreshold", obj.getCompressionContentSizeThreshold()); json.put("acceptUnmaskedFrames", obj.isAcceptUnmaskedFrames()); json.put("maxWebSocketFrameSize", obj.getMaxWebSocketFrameSize()); json.put("maxWebSocketMessageSize", obj.getMaxWebSocketMessageSize()); diff --git a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java index 69d063e0ebb..3d9fa546bcf 100755 --- a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java +++ b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java @@ -60,6 +60,11 @@ public class HttpServerOptions extends NetServerOptions { */ public static final int DEFAULT_COMPRESSION_LEVEL = 6; + /** + * Default content size threshold for compression = 0 (Netty default) + */ + public static final int DEFAULT_COMPRESSION_CONTENT_SIZE_THRESHOLD = 0; + /** * Default max WebSocket frame size = 65536 */ @@ -198,6 +203,7 @@ public class HttpServerOptions extends NetServerOptions { private boolean compressionSupported; private int compressionLevel; + private int compressionContentSizeThreshold; private List compressors; private int maxWebSocketFrameSize; private int maxWebSocketMessageSize; @@ -246,6 +252,7 @@ public HttpServerOptions(HttpServerOptions other) { super(other); this.compressionSupported = other.isCompressionSupported(); this.compressionLevel = other.getCompressionLevel(); + this.compressionContentSizeThreshold = other.getCompressionContentSizeThreshold(); this.compressors = other.compressors != null ? new ArrayList<>(other.compressors) : null; this.maxWebSocketFrameSize = other.maxWebSocketFrameSize; this.maxWebSocketMessageSize = other.maxWebSocketMessageSize; @@ -303,6 +310,7 @@ public JsonObject toJson() { private void init() { compressionSupported = DEFAULT_COMPRESSION_SUPPORTED; compressionLevel = DEFAULT_COMPRESSION_LEVEL; + compressionContentSizeThreshold = DEFAULT_COMPRESSION_CONTENT_SIZE_THRESHOLD; maxWebSocketFrameSize = DEFAULT_MAX_WEBSOCKET_FRAME_SIZE; maxWebSocketMessageSize = DEFAULT_MAX_WEBSOCKET_MESSAGE_SIZE; handle100ContinueAutomatically = DEFAULT_HANDLE_100_CONTINE_AUTOMATICALLY; @@ -588,6 +596,15 @@ public HttpServerOptions setCompressionLevel(int compressionLevel) { return this; } + public int getCompressionContentSizeThreshold() { + return compressionContentSizeThreshold; + } + + public HttpServerOptions setCompressionContentSizeThreshold(int compressionContentSizeThreshold) { + this.compressionContentSizeThreshold = compressionContentSizeThreshold; + return this; + } + /** * @return the list of compressor to use */ diff --git a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java index 55601608284..b45dd178da9 100644 --- a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java +++ b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java @@ -87,7 +87,7 @@ class HttpServerConnectionInitializer { this.exceptionHandler = exceptionHandler; this.metric = metric; this.compressionOptions = compressionOptions; - this.encodingDetector = compressionOptions != null ? new EncodingDetector(compressionOptions)::determineEncoding : null; + this.encodingDetector = compressionOptions != null ? new EncodingDetector(options.getCompressionContentSizeThreshold(), compressionOptions)::determineEncoding : null; } void configurePipeline(Channel ch, SslChannelProvider sslChannelProvider, SslContextManager sslContextManager) { @@ -265,8 +265,8 @@ void configureHttp1Handler(ChannelPipeline pipeline, SslChannelProvider sslChann private static class EncodingDetector extends HttpContentCompressor { - private EncodingDetector(CompressionOptions[] compressionOptions) { - super(compressionOptions); + private EncodingDetector(int contentSizeThreshold, CompressionOptions[] compressionOptions) { + super(contentSizeThreshold, compressionOptions); } @Override From 581739b3b2ced7f67e73c9033ecf7db60f7dc37c Mon Sep 17 00:00:00 2001 From: Will Vuong Date: Tue, 10 Sep 2024 21:00:10 -0400 Subject: [PATCH 2/3] Add javadoc and arg guard --- .../java/io/vertx/core/http/HttpServerOptions.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java index 3d9fa546bcf..24a51f57477 100755 --- a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java +++ b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java @@ -596,11 +596,22 @@ public HttpServerOptions setCompressionLevel(int compressionLevel) { return this; } + /** + * @return the compression content size threshold + */ public int getCompressionContentSizeThreshold() { return compressionContentSizeThreshold; } + /** + * Set the compression content size threshold if compression is enabled. + * If the response content size in bytes is greater than this threshold, then the response is compressed. Otherwise, it is not compressed. + * + * @param compressionContentSizeThreshold integer greater than or equal to 0. + * @return a reference to this, so the API can be used fluently + */ public HttpServerOptions setCompressionContentSizeThreshold(int compressionContentSizeThreshold) { + Arguments.require(compressionContentSizeThreshold >= 0, "compressionContentSizeThreshold must be >= 0"); this.compressionContentSizeThreshold = compressionContentSizeThreshold; return this; } From 63fd3b6dc05df01762d9ea8e55cfc58c0ac81f55 Mon Sep 17 00:00:00 2001 From: Will Vuong Date: Thu, 21 Nov 2024 10:40:18 -0500 Subject: [PATCH 3/3] Fix, add tests, update javadoc --- .../io/vertx/core/http/HttpServerOptions.java | 2 +- .../http/impl/HttpChunkContentCompressor.java | 4 +- .../impl/HttpServerConnectionInitializer.java | 4 +- .../Http1xCompressionThresholdTest.java | 97 +++++++++++++++++++ .../http/compression/HttpCompressionTest.java | 4 +- 5 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 vertx-core/src/test/java/io/vertx/tests/http/compression/Http1xCompressionThresholdTest.java diff --git a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java index 24a51f57477..493d32f31ab 100755 --- a/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java +++ b/vertx-core/src/main/java/io/vertx/core/http/HttpServerOptions.java @@ -604,7 +604,7 @@ public int getCompressionContentSizeThreshold() { } /** - * Set the compression content size threshold if compression is enabled. + * Set the compression content size threshold if compression is enabled. This is only applicable for HTTP/1.x response bodies. * If the response content size in bytes is greater than this threshold, then the response is compressed. Otherwise, it is not compressed. * * @param compressionContentSizeThreshold integer greater than or equal to 0. diff --git a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpChunkContentCompressor.java b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpChunkContentCompressor.java index cdb7ca42514..5ae963cb714 100644 --- a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpChunkContentCompressor.java +++ b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpChunkContentCompressor.java @@ -26,8 +26,8 @@ */ final class HttpChunkContentCompressor extends HttpContentCompressor { - public HttpChunkContentCompressor(CompressionOptions... compressionOptions) { - super(0, compressionOptions); + public HttpChunkContentCompressor(int contentSizeThreshold, CompressionOptions... compressionOptions) { + super(contentSizeThreshold, compressionOptions); } @Override diff --git a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java index b45dd178da9..520d68b36f3 100644 --- a/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java +++ b/vertx-core/src/main/java/io/vertx/core/http/impl/HttpServerConnectionInitializer.java @@ -52,6 +52,7 @@ class HttpServerConnectionInitializer { private final Handler exceptionHandler; private final Object metric; private final CompressionOptions[] compressionOptions; + private final int compressionContentSizeThreshold; private final Function encodingDetector; HttpServerConnectionInitializer(ContextInternal context, @@ -87,6 +88,7 @@ class HttpServerConnectionInitializer { this.exceptionHandler = exceptionHandler; this.metric = metric; this.compressionOptions = compressionOptions; + this.compressionContentSizeThreshold = options.getCompressionContentSizeThreshold(); this.encodingDetector = compressionOptions != null ? new EncodingDetector(options.getCompressionContentSizeThreshold(), compressionOptions)::determineEncoding : null; } @@ -236,7 +238,7 @@ private void configureHttp1Pipeline(ChannelPipeline pipeline, SslChannelProvider pipeline.addBefore(name, "inflater", new HttpContentDecompressor(false)); } if (options.isCompressionSupported()) { - pipeline.addBefore(name, "deflater", new HttpChunkContentCompressor(compressionOptions)); + pipeline.addBefore(name, "deflater", new HttpChunkContentCompressor(compressionContentSizeThreshold, compressionOptions)); } } diff --git a/vertx-core/src/test/java/io/vertx/tests/http/compression/Http1xCompressionThresholdTest.java b/vertx-core/src/test/java/io/vertx/tests/http/compression/Http1xCompressionThresholdTest.java new file mode 100644 index 00000000000..cd83095e7f7 --- /dev/null +++ b/vertx-core/src/test/java/io/vertx/tests/http/compression/Http1xCompressionThresholdTest.java @@ -0,0 +1,97 @@ +package io.vertx.tests.http.compression; + +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.MessageToByteEncoder; +import io.netty.handler.codec.compression.JdkZlibEncoder; +import io.netty.handler.codec.compression.ZlibWrapper; +import io.netty.util.CharsetUtil; +import io.netty.util.internal.StringUtil; +import io.vertx.core.AsyncResult; +import io.vertx.core.Handler; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.*; +import org.junit.Test; + +public class Http1xCompressionThresholdTest extends HttpCompressionTest { + + @Override + protected String encoding() { + return "gzip"; + } + + @Override + protected MessageToByteEncoder encoder() { + return new JdkZlibEncoder(ZlibWrapper.GZIP); + } + + @Override + protected HttpServerOptions createBaseServerOptions() { + return new HttpServerOptions().setPort(DEFAULT_HTTP_PORT).setHost(DEFAULT_HTTP_HOST); + } + + @Override + protected HttpClientOptions createBaseClientOptions() { + return new HttpClientOptions().setDefaultPort(DEFAULT_HTTP_PORT).setDefaultHost(DEFAULT_HTTP_HOST); + } + + @Override + protected void configureServerCompression(HttpServerOptions options) { + options.setCompressionSupported(true); + } + + @Test + public void testServerCompressionBelowThreshold() throws Exception { + // set compression threshold to be greater than the content string size so it WILL NOT be compressed + HttpServerOptions httpServerOptions = createBaseServerOptions(); + configureServerCompression(httpServerOptions); + httpServerOptions.setCompressionContentSizeThreshold(COMPRESS_TEST_STRING.length() * 2); + + doTest(httpServerOptions, onSuccess(resp -> { + // check content encoding header is not set + assertNull(resp.getHeader(HttpHeaders.CONTENT_ENCODING)); + + resp.body().onComplete(onSuccess(responseBuffer -> { + // check that the response body bytes is itself + String responseBody = responseBuffer.toString(CharsetUtil.UTF_8); + assertEquals(COMPRESS_TEST_STRING, responseBody); + testComplete(); + })); + })); + } + + @Test + public void testServerCompressionAboveThreshold() throws Exception { + // set compression threshold to be less than the content string size so it WILL be compressed + HttpServerOptions httpServerOptions = createBaseServerOptions(); + configureServerCompression(httpServerOptions); + httpServerOptions.setCompressionContentSizeThreshold(COMPRESS_TEST_STRING.length() / 2); + + doTest(httpServerOptions, onSuccess(resp -> { + // check content encoding header is set + assertEquals(encoding(), resp.getHeader(HttpHeaders.CONTENT_ENCODING)); + + resp.body().onComplete(onSuccess(responseBuffer -> { + // check that response body bytes is compressed + assertEquals(StringUtil.toHexString(compressedTestString.getBytes()), StringUtil.toHexString(responseBuffer.getBytes())); + testComplete(); + })); + })); + } + + private void doTest(HttpServerOptions httpServerOptions, Handler> handler) throws Exception { + server.close(); + server = vertx.createHttpServer(httpServerOptions); + server.requestHandler(req -> { + assertNotNull(req.headers().get(HttpHeaders.ACCEPT_ENCODING)); + req.response() + .end(Buffer.buffer(COMPRESS_TEST_STRING).toString(CharsetUtil.UTF_8)); + }); + startServer(); + client.request(new RequestOptions()) + .onComplete(onSuccess(req -> { + req.putHeader(HttpHeaders.ACCEPT_ENCODING, encoding()); + req.send().onComplete(handler); + })); + await(); + } +} diff --git a/vertx-core/src/test/java/io/vertx/tests/http/compression/HttpCompressionTest.java b/vertx-core/src/test/java/io/vertx/tests/http/compression/HttpCompressionTest.java index aa05181e00c..8a0d4ed0c3a 100644 --- a/vertx-core/src/test/java/io/vertx/tests/http/compression/HttpCompressionTest.java +++ b/vertx-core/src/test/java/io/vertx/tests/http/compression/HttpCompressionTest.java @@ -35,7 +35,7 @@ public abstract class HttpCompressionTest extends HttpTestBase { - private static final String COMPRESS_TEST_STRING = "/*\n" + + protected static final String COMPRESS_TEST_STRING = "/*\n" + " * Copyright (c) 2011-2016 The original author or authors\n" + " * ------------------------------------------------------\n" + " * All rights reserved. This program and the accompanying materials\n" + @@ -51,7 +51,7 @@ public abstract class HttpCompressionTest extends HttpTestBase { " * You may elect to redistribute this code under either of these licenses.\n" + " */"; - private Buffer compressedTestString; + protected Buffer compressedTestString; public HttpCompressionTest() { }