From 1244e246feb9419074a06bce258ca709a1701faa Mon Sep 17 00:00:00 2001 From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com> Date: Sat, 19 Oct 2024 16:43:15 +0200 Subject: [PATCH] Refactor batch insert (#28) --- .vscode/settings.json | 2 +- CHANGELOG.md | 1 + README.md | 9 +- build.gradle | 18 +- settings.gradle | 6 +- .../jdbc/BatchInsertPerformanceTest.java | 58 +++ .../org/itsallcode/jdbc/ExasolTypeTest.java | 1 + .../jdbc/NoopPreparedStatement.java | 444 ++++++++++++++++++ .../jdbc/ArgumentPreparedStatementSetter.java | 22 - .../java/org/itsallcode/jdbc/BatchInsert.java | 57 +++ .../itsallcode/jdbc/BatchInsertBuilder.java | 140 ++++++ .../ObjectArrayPreparedStatementSetter.java | 20 + .../org/itsallcode/jdbc/ParamConverter.java | 4 +- .../jdbc/RowPreparedStatementSetter.java | 23 + .../java/org/itsallcode/jdbc/SimpleBatch.java | 71 --- .../org/itsallcode/jdbc/SimpleConnection.java | 52 +- .../jdbc/SimplePreparedStatement.java | 4 +- ...eDbDialect.java => AbstractDbDialect.java} | 4 +- .../jdbc/dialect/ExasolDialect.java | 2 +- .../itsallcode/jdbc/dialect/H2Dialect.java | 2 +- .../jdbc/identifier/Identifier.java | 13 - .../jdbc/identifier/QualifiedIdentifier.java | 15 +- .../itsallcode/jdbc/resultset/RowMapper.java | 2 +- .../resultset/generic/GenericRowMapper.java | 1 + .../java/org/itsallcode/jdbc/ExampleTest.java | 22 +- .../java/org/itsallcode/jdbc/H2TypeTest.java | 3 +- .../jdbc/SimpleConnectionITest.java | 19 +- .../jdbc/identifier/IdentifierTest.java | 6 - .../ResultSetValueConverterTest.java | 5 +- 29 files changed, 832 insertions(+), 194 deletions(-) create mode 100644 src/integrationTest/java/org/itsallcode/jdbc/BatchInsertPerformanceTest.java create mode 100644 src/integrationTest/java/org/itsallcode/jdbc/NoopPreparedStatement.java delete mode 100644 src/main/java/org/itsallcode/jdbc/ArgumentPreparedStatementSetter.java create mode 100644 src/main/java/org/itsallcode/jdbc/BatchInsert.java create mode 100644 src/main/java/org/itsallcode/jdbc/BatchInsertBuilder.java create mode 100644 src/main/java/org/itsallcode/jdbc/ObjectArrayPreparedStatementSetter.java create mode 100644 src/main/java/org/itsallcode/jdbc/RowPreparedStatementSetter.java delete mode 100644 src/main/java/org/itsallcode/jdbc/SimpleBatch.java rename src/main/java/org/itsallcode/jdbc/dialect/{BaseDbDialect.java => AbstractDbDialect.java} (80%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9b7fc2c..a8f6d8c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,7 @@ "java.sources.organizeImports.staticStarThreshold": 3, "java.test.config": { "vmArgs": [ - "-Djava.util.logging.config.file=src/main/resources/logging.properties" + "-Djava.util.logging.config.file=src/test/resources/logging.properties" ] }, "sonarlint.connectedMode.project": { diff --git a/CHANGELOG.md b/CHANGELOG.md index 0251f82..28ef7e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.8.0] - unreleased - [PR #27](https://github.com/itsallcode/simple-jdbc/pull/27): Update dependencies +- [PR #28](https://github.com/itsallcode/simple-jdbc/pull/28): Refactored batch inserts (**Breaking change**) ## [0.7.1] - 2024-09-01 diff --git a/README.md b/README.md index 44a2aee..a0ca821 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Add dependency to your gradle project: ```groovy dependencies { - implementation 'org.itsallcode:simple-jdbc:0.7.1' + implementation 'org.itsallcode:simple-jdbc:0.8.0' } ``` @@ -43,8 +43,11 @@ import org.itsallcode.jdbc.resultset.SimpleResultSet; ConnectionFactory connectionFactory = ConnectionFactory.create(); try (SimpleConnection connection = connectionFactory.create("jdbc:h2:mem:", "user", "password")) { connection.executeScript(readResource("/schema.sql")); - connection.insert("NAMES", List.of("ID", "NAME"), Name::toRow, - Stream.of(new Name(1, "a"), new Name(2, "b"), new Name(3, "c"))); + connection.batchInsert(Name.class) + .into("NAMES", List.of("ID", "NAME")) + .rows(Stream.of(new Name(1, "a"), new Name(2, "b"), new Name(3, "c"))) + .mapping(Name::setPreparedStatement) + .start(); try (SimpleResultSet rs = connection.query("select * from names order by id")) { List result = rs.stream().toList(); assertEquals(3, result.size()); diff --git a/build.gradle b/build.gradle index 8677cd3..8a04dd7 100644 --- a/build.gradle +++ b/build.gradle @@ -11,10 +11,7 @@ plugins { } group 'org.itsallcode' -version = '0.7.1' - -dependencies { -} +version = '0.8.0' java { toolchain { @@ -37,6 +34,15 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +def getMockitoAgentPath() { + def mockitoAgentConfig = configurations.create("mockitoAgent") + dependencies { + mockitoAgent(libs.mockitoCore.get()) { transitive = false } + } + return mockitoAgentConfig.asPath +} +def mockitoAgentPath = getMockitoAgentPath() + testing { suites { configureEach { @@ -44,6 +50,8 @@ testing { dependencies { implementation project() implementation libs.assertj + implementation libs.mockitoJunit + runtimeOnly libs.slf4jApi runtimeOnly libs.slf4jLogger } targets { @@ -53,6 +61,7 @@ testing { testLogging.showStandardStreams = true } jvmArgs '-enableassertions' + jvmArgs "-javaagent:${mockitoAgentPath}" systemProperty 'java.util.logging.config.file', file('src/test/resources/logging.properties') } } @@ -61,7 +70,6 @@ testing { test { dependencies { implementation libs.h2 - implementation libs.mockitoJunit implementation libs.tostringverifier implementation libs.equalsverifier } diff --git a/settings.gradle b/settings.gradle index 2554f87..1ceb0ec 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,6 +10,7 @@ dependencyResolutionManagement { } versionCatalogs { libs { + version('mockito', '5.14.2') library('assertj', 'org.assertj:assertj-core:3.26.3') library('h2', 'com.h2database:h2:2.3.232') library('junitPioneer', 'org.junit-pioneer:junit-pioneer:2.2.0') @@ -17,8 +18,9 @@ dependencyResolutionManagement { library('tostringverifier', 'com.jparams:to-string-verifier:1.4.8') library('hamcrest', 'org.hamcrest:hamcrest:3.0') library('hamcrestResultSetMatcher', 'com.exasol:hamcrest-resultset-matcher:1.6.3') - library('mockito', 'org.mockito:mockito-core:5.11.0') - library('mockitoJunit', 'org.mockito:mockito-junit-jupiter:5.14.1') + library('mockitoCore', 'org.mockito', 'mockito-core').versionRef('mockito') + library('mockitoJunit', 'org.mockito', 'mockito-junit-jupiter').versionRef('mockito') + library('slf4jApi', 'org.slf4j:slf4j-api:2.0.16') library('slf4jLogger', 'org.slf4j:slf4j-jdk14:2.0.16') library('exasolJdbc', 'com.exasol:exasol-jdbc:24.1.2') library('exasolTestcontainers', 'com.exasol:exasol-testcontainers:7.1.1') diff --git a/src/integrationTest/java/org/itsallcode/jdbc/BatchInsertPerformanceTest.java b/src/integrationTest/java/org/itsallcode/jdbc/BatchInsertPerformanceTest.java new file mode 100644 index 0000000..0f61bb3 --- /dev/null +++ b/src/integrationTest/java/org/itsallcode/jdbc/BatchInsertPerformanceTest.java @@ -0,0 +1,58 @@ +package org.itsallcode.jdbc; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +import java.sql.PreparedStatement; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BatchInsertPerformanceTest { + @Mock + SimpleConnection connectionMock; + @Mock + SimplePreparedStatement preparedStatementMock; + + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void performanceTestRowStmtSetter() { + final int rowCount = 10_000_000; + testee().into("TEST", List.of("ID", "NAME")) + .mapping((row, stmt) -> { + stmt.setInt(1, row.id); + stmt.setString(2, row.name); + }).rows(generateStream(rowCount)).start(); + } + + private BatchInsertBuilder testee() { + final PreparedStatement stmt = new NoopPreparedStatement(); + when(connectionMock.prepareStatement(anyString())) + .thenReturn(new SimplePreparedStatement(null, null, stmt, "sql")); + return new BatchInsertBuilder(connectionMock::prepareStatement, Context.builder().build()); + } + + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void performanceTestObjectArray() { + final int rowCount = 10_000_000; + testee().into("TEST", List.of("ID", "NAME")) + .mapping(row -> new Object[] { row.id, row.name }).rows(generateStream(rowCount)).start(); + } + + private Stream generateStream(final int rowCount) { + return IntStream.range(0, rowCount).mapToObj(id -> new NameRow(id, "Name " + id)); + } + + private record NameRow(int id, String name) { + + } +} diff --git a/src/integrationTest/java/org/itsallcode/jdbc/ExasolTypeTest.java b/src/integrationTest/java/org/itsallcode/jdbc/ExasolTypeTest.java index 088a654..d59ebf1 100644 --- a/src/integrationTest/java/org/itsallcode/jdbc/ExasolTypeTest.java +++ b/src/integrationTest/java/org/itsallcode/jdbc/ExasolTypeTest.java @@ -22,6 +22,7 @@ class ExasolTypeTest { + @SuppressWarnings("resource") // Will be closed in @AfterAll private static final ExasolContainer container = new ExasolContainer<>("8.31.0") .withRequiredServices(ExasolService.JDBC).withReuse(true); diff --git a/src/integrationTest/java/org/itsallcode/jdbc/NoopPreparedStatement.java b/src/integrationTest/java/org/itsallcode/jdbc/NoopPreparedStatement.java new file mode 100644 index 0000000..f6f51ce --- /dev/null +++ b/src/integrationTest/java/org/itsallcode/jdbc/NoopPreparedStatement.java @@ -0,0 +1,444 @@ +package org.itsallcode.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.util.Calendar; + +public class NoopPreparedStatement implements PreparedStatement { + + @Override + public ResultSet executeQuery(final String sql) { + return null; + } + + @Override + public int executeUpdate(final String sql) { + return 0; + } + + @Override + public void close() { + } + + @Override + public int getMaxFieldSize() { + return 0; + } + + @Override + public void setMaxFieldSize(final int max) { + } + + @Override + public int getMaxRows() { + return 0; + } + + @Override + public void setMaxRows(final int max) { + } + + @Override + public void setEscapeProcessing(final boolean enable) { + } + + @Override + public int getQueryTimeout() { + return 0; + } + + @Override + public void setQueryTimeout(final int seconds) { + } + + @Override + public void cancel() { + } + + @Override + public SQLWarning getWarnings() { + return null; + } + + @Override + public void clearWarnings() { + } + + @Override + public void setCursorName(final String name) { + } + + @Override + public boolean execute(final String sql) { + return false; + } + + @Override + public ResultSet getResultSet() { + return null; + } + + @Override + public int getUpdateCount() { + return 0; + } + + @Override + public boolean getMoreResults() { + return false; + } + + @Override + public void setFetchDirection(final int direction) { + } + + @Override + public int getFetchDirection() { + return 0; + } + + @Override + public void setFetchSize(final int rows) { + + } + + @Override + public int getFetchSize() { + return 0; + } + + @Override + public int getResultSetConcurrency() { + return 0; + } + + @Override + public int getResultSetType() { + return 0; + } + + @Override + public void addBatch(final String sql) { + } + + @Override + public void clearBatch() { + } + + @Override + public int[] executeBatch() { + return new int[0]; + } + + @Override + public Connection getConnection() { + return null; + } + + @Override + public boolean getMoreResults(final int current) { + return false; + } + + @Override + public ResultSet getGeneratedKeys() { + return null; + } + + @Override + public int executeUpdate(final String sql, final int autoGeneratedKeys) { + return 0; + } + + @Override + public int executeUpdate(final String sql, final int[] columnIndexes) { + return 0; + } + + @Override + public int executeUpdate(final String sql, final String[] columnNames) { + return 0; + } + + @Override + public boolean execute(final String sql, final int autoGeneratedKeys) { + return false; + } + + @Override + public boolean execute(final String sql, final int[] columnIndexes) { + return false; + } + + @Override + public boolean execute(final String sql, final String[] columnNames) { + return false; + } + + @Override + public int getResultSetHoldability() { + return 0; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void setPoolable(final boolean poolable) { + } + + @Override + public boolean isPoolable() { + return false; + } + + @Override + public void closeOnCompletion() { + } + + @Override + public boolean isCloseOnCompletion() { + return false; + } + + @Override + public T unwrap(final Class iface) { + return null; + } + + @Override + public boolean isWrapperFor(final Class iface) { + return false; + } + + @Override + public ResultSet executeQuery() { + return null; + } + + @Override + public int executeUpdate() { + return 0; + } + + @Override + public void setNull(final int parameterIndex, final int sqlType) { + } + + @Override + public void setBoolean(final int parameterIndex, final boolean x) { + } + + @Override + public void setByte(final int parameterIndex, final byte x) { + } + + @Override + public void setShort(final int parameterIndex, final short x) { + } + + @Override + public void setInt(final int parameterIndex, final int x) { + } + + @Override + public void setLong(final int parameterIndex, final long x) { + } + + @Override + public void setFloat(final int parameterIndex, final float x) { + } + + @Override + public void setDouble(final int parameterIndex, final double x) { + } + + @Override + public void setBigDecimal(final int parameterIndex, final BigDecimal x) { + } + + @Override + public void setString(final int parameterIndex, final String x) { + } + + @Override + public void setBytes(final int parameterIndex, final byte[] x) { + } + + @Override + public void setDate(final int parameterIndex, final Date x) { + } + + @Override + public void setTime(final int parameterIndex, final Time x) { + } + + @Override + public void setTimestamp(final int parameterIndex, final Timestamp x) { + } + + @Override + public void setAsciiStream(final int parameterIndex, final InputStream x, final int length) { + } + + @SuppressWarnings("deprecation") + @Override + public void setUnicodeStream(final int parameterIndex, final InputStream x, final int length) { + } + + @Override + public void setBinaryStream(final int parameterIndex, final InputStream x, final int length) { + } + + @Override + public void clearParameters() { + } + + @Override + public void setObject(final int parameterIndex, final Object x, final int targetSqlType) { + } + + @Override + public void setObject(final int parameterIndex, final Object x) { + } + + @Override + public boolean execute() { + return false; + } + + @Override + public void addBatch() { + } + + @Override + public void setCharacterStream(final int parameterIndex, final Reader reader, final int length) { + } + + @Override + public void setRef(final int parameterIndex, final Ref x) { + } + + @Override + public void setBlob(final int parameterIndex, final Blob x) { + } + + @Override + public void setClob(final int parameterIndex, final Clob x) { + } + + @Override + public void setArray(final int parameterIndex, final Array x) { + } + + @Override + public ResultSetMetaData getMetaData() { + return null; + } + + @Override + public void setDate(final int parameterIndex, final Date x, final Calendar cal) { + } + + @Override + public void setTime(final int parameterIndex, final Time x, final Calendar cal) { + } + + @Override + public void setTimestamp(final int parameterIndex, final Timestamp x, final Calendar cal) { + } + + @Override + public void setNull(final int parameterIndex, final int sqlType, final String typeName) { + } + + @Override + public void setURL(final int parameterIndex, final URL x) { + } + + @Override + public ParameterMetaData getParameterMetaData() { + return null; + } + + @Override + public void setRowId(final int parameterIndex, final RowId x) { + } + + @Override + public void setNString(final int parameterIndex, final String value) { + } + + @Override + public void setNCharacterStream(final int parameterIndex, final Reader value, final long length) { + } + + @Override + public void setNClob(final int parameterIndex, final NClob value) { + } + + @Override + public void setClob(final int parameterIndex, final Reader reader, final long length) { + } + + @Override + public void setBlob(final int parameterIndex, final InputStream inputStream, final long length) { + } + + @Override + public void setNClob(final int parameterIndex, final Reader reader, final long length) { + } + + @Override + public void setSQLXML(final int parameterIndex, final SQLXML xmlObject) { + } + + @Override + public void setObject(final int parameterIndex, final Object x, final int targetSqlType, final int scaleOrLength) { + } + + @Override + public void setAsciiStream(final int parameterIndex, final InputStream x, final long length) { + } + + @Override + public void setBinaryStream(final int parameterIndex, final InputStream x, final long length) { + } + + @Override + public void setCharacterStream(final int parameterIndex, final Reader reader, final long length) { + } + + @Override + public void setAsciiStream(final int parameterIndex, final InputStream x) { + } + + @Override + public void setBinaryStream(final int parameterIndex, final InputStream x) { + } + + @Override + public void setCharacterStream(final int parameterIndex, final Reader reader) { + } + + @Override + public void setNCharacterStream(final int parameterIndex, final Reader value) { + } + + @Override + public void setClob(final int parameterIndex, final Reader reader) { + } + + @Override + public void setBlob(final int parameterIndex, final InputStream inputStream) { + } + + @Override + public void setNClob(final int parameterIndex, final Reader reader) { + } +} diff --git a/src/main/java/org/itsallcode/jdbc/ArgumentPreparedStatementSetter.java b/src/main/java/org/itsallcode/jdbc/ArgumentPreparedStatementSetter.java deleted file mode 100644 index 9017b52..0000000 --- a/src/main/java/org/itsallcode/jdbc/ArgumentPreparedStatementSetter.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.itsallcode.jdbc; - -import java.sql.PreparedStatement; -import java.sql.SQLException; - -class ArgumentPreparedStatementSetter implements PreparedStatementSetter { - private final Object[] args; - private final ParameterMapper mapper; - - ArgumentPreparedStatementSetter(final ParameterMapper mapper, final Object[] args) { - this.mapper = mapper; - this.args = args; - } - - public void setValues(final PreparedStatement preparedStatement) throws SQLException { - int parameterIndex = 1; - for (final Object arg : args) { - preparedStatement.setObject(parameterIndex, mapper.map(arg)); - parameterIndex++; - } - } -} diff --git a/src/main/java/org/itsallcode/jdbc/BatchInsert.java b/src/main/java/org/itsallcode/jdbc/BatchInsert.java new file mode 100644 index 0000000..256f535 --- /dev/null +++ b/src/main/java/org/itsallcode/jdbc/BatchInsert.java @@ -0,0 +1,57 @@ +package org.itsallcode.jdbc; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.logging.Logger; + +class BatchInsert implements AutoCloseable { + private static final Logger LOG = Logger.getLogger(BatchInsert.class.getName()); + + private final int maxBatchSize; + private final SimplePreparedStatement statement; + private final RowPreparedStatementSetter preparedStatementSetter; + private final Instant start; + + private int rows; + private int currentBatchSize; + + BatchInsert(final SimplePreparedStatement statement, final RowPreparedStatementSetter preparedStatementSetter, + final int maxBatchSize) { + this.preparedStatementSetter = preparedStatementSetter; + this.statement = Objects.requireNonNull(statement, "statement"); + this.maxBatchSize = maxBatchSize; + this.start = Instant.now(); + } + + void add(final T row) { + statement.setValues(stmt -> this.preparedStatementSetter.setValues(row, stmt)); + statement.addBatch(); + currentBatchSize++; + rows++; + if (rows % maxBatchSize == 0) { + executeBatch(); + } + } + + private void executeBatch() { + if (currentBatchSize == 0) { + LOG.finest("No rows added to batch, skip"); + return; + } + final Instant batchStart = Instant.now(); + final int[] result = statement.executeBatch(); + final Duration duration = Duration.between(batchStart, Instant.now()); + LOG.finest(() -> "Execute batch of %d after %d took %d ms, result length: %s".formatted(currentBatchSize, rows, + duration.toMillis(), result.length)); + currentBatchSize = 0; + } + + @Override + public void close() { + executeBatch(); + LOG.fine(() -> "Batch insert of %d rows with batch size %d completed in %s".formatted(rows, maxBatchSize, + Duration.between(start, Instant.now()))); + statement.close(); + } +} diff --git a/src/main/java/org/itsallcode/jdbc/BatchInsertBuilder.java b/src/main/java/org/itsallcode/jdbc/BatchInsertBuilder.java new file mode 100644 index 0000000..c8640f2 --- /dev/null +++ b/src/main/java/org/itsallcode/jdbc/BatchInsertBuilder.java @@ -0,0 +1,140 @@ +package org.itsallcode.jdbc; + +import static java.util.stream.Collectors.joining; + +import java.sql.PreparedStatement; +import java.util.*; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import org.itsallcode.jdbc.identifier.Identifier; + +/** + * Builder for batch inserts. + * + * @param row type + */ +public class BatchInsertBuilder { + private static final Logger LOG = Logger.getLogger(BatchInsertBuilder.class.getName()); + private static final int DEFAULT_MAX_BATCH_SIZE = 200_000; + private final Function statementFactory; + private final Context context; + private String sql; + private RowPreparedStatementSetter mapper; + private Iterator rows; + private int maxBatchSize = DEFAULT_MAX_BATCH_SIZE; + + BatchInsertBuilder(final Function statementFactory, final Context context) { + this.statementFactory = statementFactory; + this.context = context; + } + + /** + * Define table and column names used for generating the {@code INSERT} + * statement. + * + * @param tableName table name + * @param columnNames column names + * @return {@code this} for fluent programming + */ + @SuppressWarnings("java:S3242") // Using List instead of Collection to preserve column order + public BatchInsertBuilder into(final Identifier tableName, final List columnNames) { + this.sql = createInsertStatement(tableName, columnNames); + return this; + } + + /** + * Define table and column names used for generating the {@code INSERT} + * statement. + * + * @param tableName table name + * @param columnNames column names + * @return {@code this} for fluent programming + */ + @SuppressWarnings("java:S3242") // Using List instead of Collection to preserve column order + public BatchInsertBuilder into(final String tableName, final List columnNames) { + return into(Identifier.simple(tableName), columnNames.stream().map(Identifier::simple).toList()); + } + + /** + * Define {@link Stream} of rows to insert. + * + * @param rows rows to insert + * @return {@code this} for fluent programming + */ + public BatchInsertBuilder rows(final Stream rows) { + final Iterator iterator = rows.iterator(); + return rows(iterator); + } + + /** + * Define {@link Iterator} of rows to insert. + * + * @param rows rows to insert + * @return {@code this} for fluent programming + */ + public BatchInsertBuilder rows(final Iterator rows) { + this.rows = rows; + return this; + } + + /** + * Define mapping how rows are converted to {@code Object[]} for inserting. + * + * @param rowMapper row mapper + * @return {@code this} for fluent programming + */ + public BatchInsertBuilder mapping(final ParamConverter rowMapper) { + final RowPreparedStatementSetter setter = new ObjectArrayPreparedStatementSetter( + context.getParameterMapper()); + return mapping( + (final T row, final PreparedStatement preparedStatement) -> setter.setValues(rowMapper.map(row), + preparedStatement)); + } + + /** + * Define {@link RowPreparedStatementSetter} that sets values of a + * {@link PreparedStatement} for each row. + * + * @param preparedStatementSetter prepared statement setter + * @return {@code this} for fluent programming + */ + public BatchInsertBuilder mapping(final RowPreparedStatementSetter preparedStatementSetter) { + this.mapper = preparedStatementSetter; + return this; + } + + /** + * Define maximum batch size, using {@link #DEFAULT_MAX_BATCH_SIZE} as default. + * + * @param maxBatchSize maximum batch size + * @return {@code this} for fluent programming + */ + public BatchInsertBuilder maxBatchSize(final int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + return this; + } + + private static String createInsertStatement(final Identifier table, final List columnNames) { + final String columns = columnNames.stream().map(Identifier::quote).collect(joining(",")); + final String placeholders = columnNames.stream().map(n -> "?").collect(joining(",")); + return "insert into " + table.quote() + " (" + columns + ") values (" + placeholders + ")"; + } + + /** + * Start the batch insert process. + */ + public void start() { + Objects.requireNonNull(this.sql, "sql"); + Objects.requireNonNull(this.mapper, "mapper"); + Objects.requireNonNull(this.rows, "rows"); + LOG.finest(() -> "Running insert statement '" + sql + "'..."); + final SimplePreparedStatement statement = statementFactory.apply(sql); + try (BatchInsert batch = new BatchInsert<>(statement, this.mapper, this.maxBatchSize)) { + while (rows.hasNext()) { + batch.add(rows.next()); + } + } + } +} diff --git a/src/main/java/org/itsallcode/jdbc/ObjectArrayPreparedStatementSetter.java b/src/main/java/org/itsallcode/jdbc/ObjectArrayPreparedStatementSetter.java new file mode 100644 index 0000000..c0e0e6c --- /dev/null +++ b/src/main/java/org/itsallcode/jdbc/ObjectArrayPreparedStatementSetter.java @@ -0,0 +1,20 @@ +package org.itsallcode.jdbc; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +class ObjectArrayPreparedStatementSetter implements RowPreparedStatementSetter { + private final ParameterMapper mapper; + + ObjectArrayPreparedStatementSetter(final ParameterMapper mapper) { + this.mapper = mapper; + } + + public void setValues(final Object[] row, final PreparedStatement preparedStatement) throws SQLException { + int parameterIndex = 1; + for (final Object arg : row) { + preparedStatement.setObject(parameterIndex, mapper.map(arg)); + parameterIndex++; + } + } +} diff --git a/src/main/java/org/itsallcode/jdbc/ParamConverter.java b/src/main/java/org/itsallcode/jdbc/ParamConverter.java index 468203d..cf0525c 100644 --- a/src/main/java/org/itsallcode/jdbc/ParamConverter.java +++ b/src/main/java/org/itsallcode/jdbc/ParamConverter.java @@ -1,8 +1,8 @@ package org.itsallcode.jdbc; /** - * This converts a domain object to types supported by the database when - * inserting rows. + * This converts a domain object to an array of types supported by the database + * when inserting rows. * * @param row type */ diff --git a/src/main/java/org/itsallcode/jdbc/RowPreparedStatementSetter.java b/src/main/java/org/itsallcode/jdbc/RowPreparedStatementSetter.java new file mode 100644 index 0000000..fd3000f --- /dev/null +++ b/src/main/java/org/itsallcode/jdbc/RowPreparedStatementSetter.java @@ -0,0 +1,23 @@ +package org.itsallcode.jdbc; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Instances of this class allow setting values for a {@link PreparedStatement} + * for multiple rows. + * + * @param row type + */ +@FunctionalInterface +public interface RowPreparedStatementSetter { + /** + * Set values for the given prepared statement. + * + * + * @param row the row for which to set values + * @param preparedStatement the prepared statement + * @throws SQLException if setting values fails + */ + void setValues(T row, PreparedStatement preparedStatement) throws SQLException; +} diff --git a/src/main/java/org/itsallcode/jdbc/SimpleBatch.java b/src/main/java/org/itsallcode/jdbc/SimpleBatch.java deleted file mode 100644 index 2c433ce..0000000 --- a/src/main/java/org/itsallcode/jdbc/SimpleBatch.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.itsallcode.jdbc; - -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.logging.Logger; - -import org.itsallcode.jdbc.SimpleParameterMetaData.Parameter; - -class SimpleBatch implements AutoCloseable { - private static final Logger LOG = Logger.getLogger(SimpleBatch.class.getName()); - private static final int BATCH_SIZE = 200_000; - - private final SimplePreparedStatement statement; - private final Context context; - private final List parameterMetadata; - - private int rows; - private int currentBatchSize; - - SimpleBatch(final SimplePreparedStatement statement, final Context context) { - this.statement = Objects.requireNonNull(statement, "statement"); - this.context = Objects.requireNonNull(context, "context"); - this.parameterMetadata = statement.getParameterMetadata().parameters(); - } - - @SuppressWarnings("java:S923") // Varargs required - SimpleBatch add(final Object... args) { - validateParameters(args); - return add(new ArgumentPreparedStatementSetter(context.getParameterMapper(), args)); - } - - @SuppressWarnings("java:S923") // Varargs required - private void validateParameters(final Object... args) { - if (args.length != this.parameterMetadata.size()) { - throw new IllegalStateException( - "Expected " + this.parameterMetadata.size() + " arguments but got " + args.length + ": " - + Arrays.toString(args) + ", " + parameterMetadata); - } - } - - private SimpleBatch add(final PreparedStatementSetter preparedStatementSetter) { - statement.setValues(preparedStatementSetter); - statement.addBatch(); - currentBatchSize++; - rows++; - if (rows % BATCH_SIZE == 0) { - executeBatch(); - } - return this; - } - - @Override - public void close() { - executeBatch(); - statement.close(); - } - - private void executeBatch() { - if (currentBatchSize == 0) { - LOG.fine("No rows added to batch, skip"); - return; - } - final Instant start = Instant.now(); - statement.executeBatch(); - final Duration duration = Duration.between(start, Instant.now()); - LOG.fine(() -> "Execute batch of " + currentBatchSize + " after " + rows + " took " + duration.toMillis() - + " ms"); - currentBatchSize = 0; - } -} diff --git a/src/main/java/org/itsallcode/jdbc/SimpleConnection.java b/src/main/java/org/itsallcode/jdbc/SimpleConnection.java index 85c241d..058c35b 100644 --- a/src/main/java/org/itsallcode/jdbc/SimpleConnection.java +++ b/src/main/java/org/itsallcode/jdbc/SimpleConnection.java @@ -1,15 +1,13 @@ package org.itsallcode.jdbc; import static java.util.function.Predicate.not; -import static java.util.stream.Collectors.joining; import java.sql.*; -import java.util.*; +import java.util.Arrays; +import java.util.Objects; import java.util.logging.Logger; -import java.util.stream.Stream; import org.itsallcode.jdbc.dialect.DbDialect; -import org.itsallcode.jdbc.identifier.Identifier; import org.itsallcode.jdbc.resultset.*; import org.itsallcode.jdbc.resultset.generic.Row; @@ -98,51 +96,19 @@ public SimpleResultSet query(final String sql, final PreparedStatementSet return statement.executeQuery(ContextRowMapper.create(rowMapper)); } - private SimplePreparedStatement prepareStatement(final String sql) { + SimplePreparedStatement prepareStatement(final String sql) { return new SimplePreparedStatement(context, dialect, prepare(sql), sql); } /** - * Insert rows into a table using batch operation. + * Create a batch insert builder * - * @param generic row type - * @param table table name - * @param columnNames column names - * @param rowMapper a mapper to convert each column - * @param rows a stream of rows to insert + * @param rowType row type + * @param row type + * @return batch insert builder */ - public void insert(final String table, final List columnNames, final ParamConverter rowMapper, - final Stream rows) { - insert(Identifier.simple(table), columnNames.stream().map(Identifier::simple).toList(), rowMapper, rows); - } - - /** - * Insert rows into a table using batch operation. - * - * @param generic row type - * @param table table name - * @param columnNames column names - * @param rowMapper a mapper to convert each column - * @param rows a stream of rows to insert - */ - public void insert(final Identifier table, final List columnNames, - final ParamConverter rowMapper, final Stream rows) { - insert(createInsertStatement(table, columnNames), rowMapper, rows); - } - - private static String createInsertStatement(final Identifier table, final List columnNames) { - final String columns = columnNames.stream().map(Identifier::quote).collect(joining(",")); - final String placeholders = columnNames.stream().map(n -> "?").collect(joining(",")); - return "insert into " + table.quote() + " (" + columns + ") values (" + placeholders + ")"; - } - - void insert(final String sql, final ParamConverter paramConverter, final Stream rows) { - LOG.fine(() -> "Running insert statement '" + sql + "'..."); - try (SimpleBatch batch = new SimpleBatch(prepareStatement(sql), context)) { - rows.map(paramConverter::map).forEach(batch::add); - } finally { - rows.close(); - } + public BatchInsertBuilder batchInsert(final Class rowType) { + return new BatchInsertBuilder<>(this::prepareStatement, context); } private PreparedStatement prepare(final String sql) { diff --git a/src/main/java/org/itsallcode/jdbc/SimplePreparedStatement.java b/src/main/java/org/itsallcode/jdbc/SimplePreparedStatement.java index 6d64eab..ac0803a 100644 --- a/src/main/java/org/itsallcode/jdbc/SimplePreparedStatement.java +++ b/src/main/java/org/itsallcode/jdbc/SimplePreparedStatement.java @@ -51,9 +51,9 @@ void setValues(final PreparedStatementSetter preparedStatementSetter) { } - void executeBatch() { + int[] executeBatch() { try { - statement.executeBatch(); + return statement.executeBatch(); } catch (final SQLException e) { throw new UncheckedSQLException("Error executing batch sql '" + sql + "'", e); } diff --git a/src/main/java/org/itsallcode/jdbc/dialect/BaseDbDialect.java b/src/main/java/org/itsallcode/jdbc/dialect/AbstractDbDialect.java similarity index 80% rename from src/main/java/org/itsallcode/jdbc/dialect/BaseDbDialect.java rename to src/main/java/org/itsallcode/jdbc/dialect/AbstractDbDialect.java index a4156e7..fe70377 100644 --- a/src/main/java/org/itsallcode/jdbc/dialect/BaseDbDialect.java +++ b/src/main/java/org/itsallcode/jdbc/dialect/AbstractDbDialect.java @@ -5,7 +5,7 @@ /** * Base class for implementing a {@link DbDialect}. */ -public abstract class BaseDbDialect implements DbDialect { +public abstract class AbstractDbDialect implements DbDialect { private final String jdbcUrlPrefix; /** @@ -13,7 +13,7 @@ public abstract class BaseDbDialect implements DbDialect { * * @param jdbcUrlPrefix the JDBC URL prefix supported by this dialect */ - protected BaseDbDialect(final String jdbcUrlPrefix) { + protected AbstractDbDialect(final String jdbcUrlPrefix) { this.jdbcUrlPrefix = jdbcUrlPrefix.toLowerCase(Locale.ROOT); } diff --git a/src/main/java/org/itsallcode/jdbc/dialect/ExasolDialect.java b/src/main/java/org/itsallcode/jdbc/dialect/ExasolDialect.java index 8230b0a..8729764 100644 --- a/src/main/java/org/itsallcode/jdbc/dialect/ExasolDialect.java +++ b/src/main/java/org/itsallcode/jdbc/dialect/ExasolDialect.java @@ -5,7 +5,7 @@ /** * Dialect for the Exasol database. */ -public class ExasolDialect extends BaseDbDialect { +public class ExasolDialect extends AbstractDbDialect { /** * Create a new instance. diff --git a/src/main/java/org/itsallcode/jdbc/dialect/H2Dialect.java b/src/main/java/org/itsallcode/jdbc/dialect/H2Dialect.java index 5accca9..390c4b1 100644 --- a/src/main/java/org/itsallcode/jdbc/dialect/H2Dialect.java +++ b/src/main/java/org/itsallcode/jdbc/dialect/H2Dialect.java @@ -8,7 +8,7 @@ /** * DB dialect for the H2 database. */ -public class H2Dialect extends BaseDbDialect { +public class H2Dialect extends AbstractDbDialect { /** * Create a new instance. diff --git a/src/main/java/org/itsallcode/jdbc/identifier/Identifier.java b/src/main/java/org/itsallcode/jdbc/identifier/Identifier.java index f300392..3a62001 100644 --- a/src/main/java/org/itsallcode/jdbc/identifier/Identifier.java +++ b/src/main/java/org/itsallcode/jdbc/identifier/Identifier.java @@ -1,7 +1,5 @@ package org.itsallcode.jdbc.identifier; -import java.util.Arrays; - /** * Represents a database identifier, e.g. of a table or a schema. */ @@ -25,15 +23,4 @@ public interface Identifier { static Identifier simple(final String id) { return SimpleIdentifier.of(id); } - - /** - * Create a new {@link QualifiedIdentifier} from the given parts. - * - * @param id parts of the ID - * @return a new {@link QualifiedIdentifier} - */ - @SuppressWarnings("java:S923") // Varargs required - static Identifier qualified(final String... id) { - return QualifiedIdentifier.of(Arrays.stream(id).map(SimpleIdentifier::of).toArray(SimpleIdentifier[]::new)); - } } diff --git a/src/main/java/org/itsallcode/jdbc/identifier/QualifiedIdentifier.java b/src/main/java/org/itsallcode/jdbc/identifier/QualifiedIdentifier.java index 753c673..3e53369 100644 --- a/src/main/java/org/itsallcode/jdbc/identifier/QualifiedIdentifier.java +++ b/src/main/java/org/itsallcode/jdbc/identifier/QualifiedIdentifier.java @@ -7,8 +7,10 @@ /** * A qualified identifier, e.g. table name and schema name. + * + * @param id list of identifiers */ -record QualifiedIdentifier(List id) implements Identifier { +public record QualifiedIdentifier(List id) implements Identifier { /** * Create a new qualified identifier. @@ -21,6 +23,17 @@ public static Identifier of(final Identifier... ids) { return new QualifiedIdentifier(asList(ids)); } + /** + * Create a new qualified identifier. + * + * @param ids the simple IDs + * @return a new instance + */ + @SuppressWarnings("java:S923") // Varargs required + public static Identifier of(final String... ids) { + return of(asList(ids).stream().map(Identifier::simple).toArray(Identifier[]::new)); + } + @Override public String toString() { return quote(); diff --git a/src/main/java/org/itsallcode/jdbc/resultset/RowMapper.java b/src/main/java/org/itsallcode/jdbc/resultset/RowMapper.java index 4cd3806..38fcd4d 100644 --- a/src/main/java/org/itsallcode/jdbc/resultset/RowMapper.java +++ b/src/main/java/org/itsallcode/jdbc/resultset/RowMapper.java @@ -19,4 +19,4 @@ public interface RowMapper { * @throws SQLException if accessing the result set fails */ T mapRow(ResultSet resultSet, int rowNum) throws SQLException; -} \ No newline at end of file +} diff --git a/src/main/java/org/itsallcode/jdbc/resultset/generic/GenericRowMapper.java b/src/main/java/org/itsallcode/jdbc/resultset/generic/GenericRowMapper.java index 1008f10..c8ea63d 100644 --- a/src/main/java/org/itsallcode/jdbc/resultset/generic/GenericRowMapper.java +++ b/src/main/java/org/itsallcode/jdbc/resultset/generic/GenericRowMapper.java @@ -78,6 +78,7 @@ private static Object getValue(final ResultSet resultSet, final ColumnMetaData c * @param generic row type */ @FunctionalInterface + @SuppressWarnings("java:S1711") // Explicit interface instead of generic Function<> public interface ColumnValuesConverter { /** * Convert a single row. diff --git a/src/test/java/org/itsallcode/jdbc/ExampleTest.java b/src/test/java/org/itsallcode/jdbc/ExampleTest.java index d62c53d..93b83e6 100644 --- a/src/test/java/org/itsallcode/jdbc/ExampleTest.java +++ b/src/test/java/org/itsallcode/jdbc/ExampleTest.java @@ -5,6 +5,8 @@ import java.io.*; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; @@ -13,19 +15,25 @@ import org.junit.jupiter.api.Test; class ExampleTest { + record Name(int id, String name) { + static void setPreparedStatement(final Name row, final PreparedStatement stmt) throws SQLException { + stmt.setInt(1, row.id); + stmt.setString(2, row.name); + } + } + @Test void example() { - record Name(int id, String name) { - Object[] toRow() { - return new Object[] { id, name }; - } - } final ConnectionFactory connectionFactory = ConnectionFactory .create(Context.builder().build()); try (SimpleConnection connection = connectionFactory.create("jdbc:h2:mem:", "user", "password")) { connection.executeScript(readResource("/schema.sql")); - connection.insert("NAMES", List.of("ID", "NAME"), Name::toRow, - Stream.of(new Name(1, "a"), new Name(2, "b"), new Name(3, "c"))); + connection.batchInsert(Name.class) + .into("NAMES", List.of("ID", "NAME")) + .rows(Stream.of(new Name(1, "a"), new Name(2, "b"), new Name(3, "c"))) + .mapping(Name::setPreparedStatement) + .start(); + try (SimpleResultSet rs = connection.query("select * from names order by id")) { final List result = rs.stream().toList(); assertEquals(3, result.size()); diff --git a/src/test/java/org/itsallcode/jdbc/H2TypeTest.java b/src/test/java/org/itsallcode/jdbc/H2TypeTest.java index 78c2b99..4e872dc 100644 --- a/src/test/java/org/itsallcode/jdbc/H2TypeTest.java +++ b/src/test/java/org/itsallcode/jdbc/H2TypeTest.java @@ -79,7 +79,8 @@ void preparedStatementSetParameter(final TypeTest test) { try (final SimpleConnection connection = H2TestFixture.createMemConnection(); final SimpleResultSet result = connection .query("select ?", - (preparedStatement) -> preparedStatement.setObject(1, value), + preparedStatement -> preparedStatement.setObject(1, + value), (resultSet, rowNum) -> resultSet .getObject(1, value.getClass()))) { assertThat(result.toList()).containsExactly(value); diff --git a/src/test/java/org/itsallcode/jdbc/SimpleConnectionITest.java b/src/test/java/org/itsallcode/jdbc/SimpleConnectionITest.java index 689e6e9..43a8c5b 100644 --- a/src/test/java/org/itsallcode/jdbc/SimpleConnectionITest.java +++ b/src/test/java/org/itsallcode/jdbc/SimpleConnectionITest.java @@ -10,7 +10,6 @@ import java.util.*; import java.util.stream.Stream; -import org.itsallcode.jdbc.identifier.Identifier; import org.itsallcode.jdbc.resultset.*; import org.itsallcode.jdbc.resultset.generic.Row; import org.junit.jupiter.api.Test; @@ -196,7 +195,9 @@ void executeQueryOnlyOneIteratorAllowed() { void batchInsertEmptyInput() { try (SimpleConnection connection = H2TestFixture.createMemConnection()) { connection.executeScript("CREATE TABLE TEST(ID INT, NAME VARCHAR(255))"); - connection.insert("insert into test (id, name) values (?, ?)", ParamConverter.identity(), Stream.empty()); + connection.batchInsert(Object[].class).into("TEST", List.of("ID", "NAME")) + .mapping(ParamConverter.identity()) + .rows(Stream.empty()).start(); final List result = connection.query("select * from test").stream().toList(); assertThat(result).isEmpty(); @@ -207,8 +208,10 @@ void batchInsertEmptyInput() { void batchInsert() { try (SimpleConnection connection = H2TestFixture.createMemConnection()) { connection.executeScript("CREATE TABLE TEST(ID INT, NAME VARCHAR(255))"); - connection.insert("insert into test (id, name) values (?, ?)", ParamConverter.identity(), - Stream.of(new Object[] { 1, "a" }, new Object[] { 2, "b" }, new Object[] { 3, "c" })); + connection.batchInsert(Object[].class).into("TEST", List.of("ID", "NAME")) + .mapping(ParamConverter.identity()).rows( + Stream.of(new Object[] { 1, "a" }, new Object[] { 2, "b" }, new Object[] { 3, "c" })) + .start(); final List result = connection.query("select count(*) from test").stream().toList(); assertAll( @@ -222,9 +225,11 @@ void batchInsert() { void insert() { try (SimpleConnection connection = H2TestFixture.createMemConnection()) { connection.executeScript("CREATE TABLE TEST(ID INT, NAME VARCHAR(255))"); - connection.insert(Identifier.simple("TEST"), List.of(Identifier.simple("ID"), Identifier.simple("NAME")), - ParamConverter.identity(), - Stream.of(new Object[] { 1, "a" }, new Object[] { 2, "b" }, new Object[] { 3, "c" })); + connection.batchInsert(Object[].class).into("TEST", List.of("ID", "NAME")) + .mapping(ParamConverter.identity()) + .rows( + Stream.of(new Object[] { 1, "a" }, new Object[] { 2, "b" }, new Object[] { 3, "c" })) + .start(); final List> result = connection.query("select * from test").stream() .map(row -> row.columnValues().stream().map(value -> value.value()).toList()).toList(); diff --git a/src/test/java/org/itsallcode/jdbc/identifier/IdentifierTest.java b/src/test/java/org/itsallcode/jdbc/identifier/IdentifierTest.java index dcf320f..f4460f8 100644 --- a/src/test/java/org/itsallcode/jdbc/identifier/IdentifierTest.java +++ b/src/test/java/org/itsallcode/jdbc/identifier/IdentifierTest.java @@ -10,10 +10,4 @@ void simple() { final Identifier id = Identifier.simple("id"); assertThat(id).isNotNull().hasToString("\"id\""); } - - @Test - void qualified() { - final Identifier id = Identifier.qualified("id1", "id2"); - assertThat(id).isNotNull().hasToString("\"id1\".\"id2\""); - } } diff --git a/src/test/java/org/itsallcode/jdbc/resultset/ResultSetValueConverterTest.java b/src/test/java/org/itsallcode/jdbc/resultset/ResultSetValueConverterTest.java index 4d3aad8..246cb14 100644 --- a/src/test/java/org/itsallcode/jdbc/resultset/ResultSetValueConverterTest.java +++ b/src/test/java/org/itsallcode/jdbc/resultset/ResultSetValueConverterTest.java @@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import org.itsallcode.jdbc.resultset.generic.ColumnMetaData; @@ -21,7 +20,7 @@ class ResultSetValueConverterTest { ResultSet resultSetMock; @Test - void emptyInputNoConverterFoundForIndex() throws SQLException { + void emptyInputNoConverterFoundForIndex() { final ResultSetValueConverter converter = testee(emptyList(), emptyList()); assertThatThrownBy(() -> converter.getObject(resultSetMock, 1)) .isInstanceOf(IllegalStateException.class) @@ -29,7 +28,7 @@ void emptyInputNoConverterFoundForIndex() throws SQLException { } @Test - void emptyInputNoConverterFoundForLabel() throws SQLException { + void emptyInputNoConverterFoundForLabel() { final ResultSetValueConverter converter = testee(emptyList(), emptyList()); assertThatThrownBy(() -> converter.getObject(resultSetMock, "label")) .isInstanceOf(IllegalStateException.class)