-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: disable interval-based auto-flushes by default
the connector has its own mechanism to flush on inactivity, we don't need the client to do this as well. we disable it only when it's not set explicitly. since if a user sets an explicit flush interval then we can assume they know what they are doing.
- Loading branch information
Showing
6 changed files
with
140 additions
and
18 deletions.
There are no files selected for viewing
66 changes: 66 additions & 0 deletions
66
connector/src/main/java/io/questdb/kafka/ClientConfUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package io.questdb.kafka; | ||
|
|
||
| import io.questdb.client.impl.ConfStringParser; | ||
| import io.questdb.std.Chars; | ||
| import io.questdb.std.str.StringSink; | ||
|
|
||
| final class ClientConfUtils { | ||
| private ClientConfUtils() { | ||
| } | ||
|
|
||
| static boolean patchConfStr(String confStr, StringSink sink) { | ||
| int pos = ConfStringParser.of(confStr, sink); | ||
| if (pos < 0) { | ||
| sink.clear(); | ||
| sink.put(confStr); | ||
| return false; | ||
| } | ||
|
|
||
| boolean isHttpTransport = Chars.equals(sink, "http") || Chars.equals(sink, "https"); | ||
| boolean intervalFlushSetExplicitly = false; | ||
| boolean flushesDisabled = false; | ||
| boolean parseError = false; | ||
| boolean hasAtLeastOneParam = false; | ||
|
|
||
| // disable interval based flushes | ||
| // unless they are explicitly set or auto_flush is entirely off | ||
| // why? the connector has its own mechanism to flush data in a timely manner | ||
| while (ConfStringParser.hasNext(confStr, pos)) { | ||
| hasAtLeastOneParam = true; | ||
| pos = ConfStringParser.nextKey(confStr, pos, sink); | ||
| if (pos < 0) { | ||
| parseError = true; | ||
| break; | ||
| } | ||
| if (Chars.equals(sink, "auto_flush_interval")) { | ||
| intervalFlushSetExplicitly = true; | ||
| pos = ConfStringParser.value(confStr, pos, sink); | ||
| } else if (Chars.equals(sink, "auto_flush")) { | ||
| pos = ConfStringParser.value(confStr, pos, sink); | ||
| flushesDisabled = Chars.equals(sink, "off"); | ||
| } else { | ||
| pos = ConfStringParser.value(confStr, pos, sink); // skip other values | ||
| } | ||
| if (pos < 0) { | ||
| parseError = true; | ||
| break; | ||
| } | ||
| } | ||
| sink.clear(); | ||
| sink.put(confStr); | ||
| if (!parseError // we don't want to mess with the config if there was a parse error | ||
| && isHttpTransport // we only want to patch http transport | ||
| && !flushesDisabled // if auto-flush is disabled we don't need to do anything | ||
| && !intervalFlushSetExplicitly // if auto_flush_interval is set explicitly we don't want to override it | ||
| && hasAtLeastOneParam // no parameter is also an error since at least address should be set. we let client throw exception in this case | ||
| ) { | ||
| // if everything is ok, we set auto_flush_interval to max value | ||
| // this will effectively disable interval based flushes | ||
| // and the connector will flush data only when it is told to do so by Connector | ||
| // or if a row count limit is reached | ||
| sink.put("auto_flush_interval=").put(Integer.MAX_VALUE).put(';'); | ||
| } | ||
|
|
||
| return isHttpTransport; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
connector/src/test/java/io/questdb/kafka/ClientConfUtilsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package io.questdb.kafka; | ||
|
|
||
| import io.questdb.std.Chars; | ||
| import io.questdb.std.str.StringSink; | ||
| import org.junit.Test; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.*; | ||
|
|
||
| public class ClientConfUtilsTest { | ||
|
|
||
| @Test | ||
| public void testHttpTransportIsResolved() { | ||
| StringSink sink = new StringSink(); | ||
| assertTrue(ClientConfUtils.patchConfStr("http::addr=localhost:9000;", sink)); | ||
| assertTrue(ClientConfUtils.patchConfStr("https::addr=localhost:9000;", sink)); | ||
| assertTrue(ClientConfUtils.patchConfStr("https::addr=localhost:9000;", sink)); | ||
| assertFalse(ClientConfUtils.patchConfStr("tcp::addr=localhost:9000;", sink)); | ||
| assertFalse(ClientConfUtils.patchConfStr("tcps::addr=localhost:9000;", sink)); | ||
| } | ||
|
|
||
| @Test | ||
| public void testHttpTransportTimeBasedFlushesDisabledByDefault() { | ||
| assertConfStringIsPatched("http::addr=localhost:9000;"); | ||
| assertConfStringIsPatched("https::addr=localhost:9000;foo=bar;"); | ||
| assertConfStringIsPatched("https::addr=localhost:9000;auto_flush_rows=1;"); | ||
| assertConfStringIsPatched("https::addr=localhost:9000;auto_flush=on;"); | ||
|
|
||
| assertConfStringIsNotPatched("https::addr=localhost:9000;foo=bar;auto_flush_interval=100;"); | ||
| assertConfStringIsNotPatched("https::addr=localhost:9000;foo=bar;auto_flush=off;"); | ||
| assertConfStringIsNotPatched("https::addr=localhost:9000;foo=bar"); | ||
| assertConfStringIsNotPatched("https::addr"); | ||
| assertConfStringIsNotPatched("https"); | ||
| assertConfStringIsNotPatched("tcp::addr=localhost:9000;"); | ||
| assertConfStringIsNotPatched("tcps::addr=localhost:9000;foo=bar;"); | ||
| assertConfStringIsNotPatched("tcps::addr=localhost:9000;auto_flush_rows=1;"); | ||
| assertConfStringIsNotPatched("tcps::addr=localhost:9000;auto_flush=on;"); | ||
| assertConfStringIsNotPatched("unknown::addr=localhost:9000;auto_flush=on;"); | ||
| } | ||
|
|
||
| private static void assertConfStringIsPatched(String confStr) { | ||
| StringSink sink = new StringSink(); | ||
| ClientConfUtils.patchConfStr(confStr, sink); | ||
|
|
||
| String expected = confStr + "auto_flush_interval=" + Integer.MAX_VALUE + ";"; | ||
| assertTrue(Chars.equals(expected, sink), "Conf string = " + confStr + ", expected = " + expected + ", actual = " + sink); | ||
| } | ||
|
|
||
| private static void assertConfStringIsNotPatched(String confStr) { | ||
| StringSink sink = new StringSink(); | ||
| ClientConfUtils.patchConfStr(confStr, sink); | ||
|
|
||
| assertEquals(confStr, sink.toString()); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters