From 93e98ea90a2f3d0f20eb6761ffd4c8025a0c3730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 29 Oct 2024 18:57:19 +0100 Subject: [PATCH 01/29] return toStringResult for getObject and getBytes called on structured type --- .../client/core/SFArrowResultSet.java | 12 ++---------- .../client/core/arrow/ArrayConverter.java | 7 ++++++- .../client/core/arrow/MapConverter.java | 11 ++++++----- .../client/core/arrow/StructConverter.java | 7 ++++++- .../client/core/arrow/VectorTypeConverter.java | 5 +++++ ...TypesGetStringArrowJsonCompatibilityIT.java | 18 ++++++++++++++++++ .../StructuredTypesGetStringBaseIT.java | 17 ++++++++++++++++- 7 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 14f21e8d1..7ebca741b 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -573,16 +573,8 @@ public Object getObject(int columnIndex) throws SFException { converter.setTreatNTZAsUTC(treatNTZAsUTC); converter.setUseSessionTimezone(useSessionTimezone); converter.setSessionTimeZone(sessionTimeZone); - Object obj = converter.toObject(index); - boolean isStructuredType = resultSetMetaData.isStructuredTypeColumn(columnIndex); - if (type == Types.STRUCT && isStructuredType) { - if (converter instanceof VarCharConverter) { - return createJsonSqlInput(columnIndex, obj); - } else if (converter instanceof StructConverter) { - return createArrowSqlInput(columnIndex, (Map) obj); - } - } - return obj; + + return converter.toString(index); } private Object createJsonSqlInput(int columnIndex, Object obj) throws SFException { diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 48b8fa083..dfafb14dc 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -19,7 +19,12 @@ public ArrayConverter(ListVector valueVector, int vectorIndex, DataConversionCon @Override public Object toObject(int index) throws SFException { - return vector.getObject(index); + return toString(index); + } + + @Override + public byte[] toBytes(int index) throws SFException { + return toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 8ef1cdccf..4b982ad84 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -22,11 +22,12 @@ public MapConverter(MapVector valueVector, int columnIndex, DataConversionContex @Override public Object toObject(int index) throws SFException { - List> entriesList = - (List>) vector.getObject(index); - return entriesList.stream() - .collect( - Collectors.toMap(entry -> entry.get("key").toString(), entry -> entry.get("value"))); + return toString(index); + } + + @Override + public byte[] toBytes(int index) throws SFException { + return toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 4c0516c51..fea1136ba 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -21,7 +21,12 @@ public StructConverter(StructVector vector, int columnIndex, DataConversionConte @Override public Object toObject(int index) throws SFException { - return structVector.getObject(index); + return toString(index); + } + + @Override + public byte[] toBytes(int index) throws SFException { + return toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index ae7a492a0..b95b23a6f 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -21,6 +21,11 @@ public Object toObject(int index) throws SFException { return vector.getObject(index); } + @Override + public byte[] toBytes(int index) throws SFException { + return toString(index).getBytes(); + } + @Override public String toString(int index) throws SFException { List object = vector.getObject(index); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index 352d2b1a4..c3677b54f 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -62,6 +62,24 @@ public void testRunAsGetString() throws SQLException { (resultSet) -> assertGetStringIsCompatible(resultSet, expectedStructureTypeRepresentation)); } + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testRunAsGetObject() throws SQLException { + withFirstRow( + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); + } + + @Test + @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) + public void testRunAsGetBytes() throws SQLException { + withFirstRow( + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); + } + @Parameterized.Parameters(name = "format={0},sql={1}") public static Collection data() { Map samples = new LinkedHashMap<>(); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index d9d5c15e2..cabb81a3b 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -50,12 +51,26 @@ protected static Connection initConnection(ResultSetFormatType queryResultFormat return conn; } + protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) + throws SQLException { + String result = new String(resultSet.getBytes(1), StandardCharsets.UTF_8); + TestUtil.assertEqualsIgnoringWhitespace(expected, result); + } + protected void assertGetStringIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + throws SQLException { String result = resultSet.getString(1); TestUtil.assertEqualsIgnoringWhitespace(expected, result); } + protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) + throws SQLException { + String result = resultSet.getObject(1, String.class); + String resultCasted = (String) resultSet.getObject(1); + TestUtil.assertEqualsIgnoringWhitespace(expected, result); + TestUtil.assertEqualsIgnoringWhitespace(expected, resultCasted); + } + protected void withFirstRow( Connection connection, String sqlText, ThrowingConsumer consumer) throws SQLException { From 3ae60f58eec7b4c5b9dd41dcf7af9ef5b4b133da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 29 Oct 2024 19:03:36 +0100 Subject: [PATCH 02/29] fix formatting --- .../net/snowflake/client/core/SFArrowResultSet.java | 1 - .../snowflake/client/core/arrow/MapConverter.java | 3 --- ...cturedTypesGetStringArrowJsonCompatibilityIT.java | 12 ++++++------ .../StructuredTypesGetStringBaseIT.java | 4 ++-- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 7ebca741b..efbc304a2 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -26,7 +26,6 @@ import java.util.stream.Stream; import net.snowflake.client.core.arrow.ArrayConverter; import net.snowflake.client.core.arrow.ArrowVectorConverter; -import net.snowflake.client.core.arrow.StructConverter; import net.snowflake.client.core.arrow.VarCharConverter; import net.snowflake.client.core.arrow.VectorTypeConverter; import net.snowflake.client.core.json.Converters; diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 4b982ad84..db68577bc 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -1,7 +1,5 @@ package net.snowflake.client.core.arrow; -import java.util.List; -import java.util.stream.Collectors; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.arrow.tostringhelpers.ArrowObjectStringRepresentationBuilder; @@ -9,7 +7,6 @@ import net.snowflake.client.jdbc.SnowflakeType; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.util.JsonStringHashMap; public class MapConverter extends AbstractArrowVectorConverter { diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index c3677b54f..66ccb0096 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -66,18 +66,18 @@ public void testRunAsGetString() throws SQLException { @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testRunAsGetObject() throws SQLException { withFirstRow( - connections.get(queryResultFormat), - selectSql, - (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); } @Test @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testRunAsGetBytes() throws SQLException { withFirstRow( - connections.get(queryResultFormat), - selectSql, - (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); } @Parameterized.Parameters(name = "format={0},sql={1}") diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index cabb81a3b..9c8b9c29a 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -52,13 +52,13 @@ protected static Connection initConnection(ResultSetFormatType queryResultFormat } protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + throws SQLException { String result = new String(resultSet.getBytes(1), StandardCharsets.UTF_8); TestUtil.assertEqualsIgnoringWhitespace(expected, result); } protected void assertGetStringIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + throws SQLException { String result = resultSet.getString(1); TestUtil.assertEqualsIgnoringWhitespace(expected, result); } From a55e26a9bcf681c3b26f5a085f74c32e58941623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 29 Oct 2024 19:45:12 +0100 Subject: [PATCH 03/29] remove getBytes implementation from structured types converters --- .../java/net/snowflake/client/core/arrow/ArrayConverter.java | 5 ----- .../java/net/snowflake/client/core/arrow/MapConverter.java | 5 ----- .../net/snowflake/client/core/arrow/StructConverter.java | 5 ----- .../net/snowflake/client/core/arrow/VectorTypeConverter.java | 5 ----- 4 files changed, 20 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index dfafb14dc..13ebce18f 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -22,11 +22,6 @@ public Object toObject(int index) throws SFException { return toString(index); } - @Override - public byte[] toBytes(int index) throws SFException { - return toString(index).getBytes(); - } - @Override public String toString(int index) throws SFException { FieldVector vectorUnpacked = vector.getChildrenFromFields().get(0); diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index db68577bc..c869a6205 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -22,11 +22,6 @@ public Object toObject(int index) throws SFException { return toString(index); } - @Override - public byte[] toBytes(int index) throws SFException { - return toString(index).getBytes(); - } - @Override public String toString(int index) throws SFException { ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index fea1136ba..db9554660 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -24,11 +24,6 @@ public Object toObject(int index) throws SFException { return toString(index); } - @Override - public byte[] toBytes(int index) throws SFException { - return toString(index).getBytes(); - } - @Override public String toString(int index) throws SFException { ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index b95b23a6f..ae7a492a0 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -21,11 +21,6 @@ public Object toObject(int index) throws SFException { return vector.getObject(index); } - @Override - public byte[] toBytes(int index) throws SFException { - return toString(index).getBytes(); - } - @Override public String toString(int index) throws SFException { List object = vector.getObject(index); From 2e86075d5afe02b89d3f9ce18848e262f98a8143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 29 Oct 2024 20:41:22 +0100 Subject: [PATCH 04/29] remove getBytes, add null check to getObject --- .../java/net/snowflake/client/core/SFArrowResultSet.java | 8 ++++++-- .../net/snowflake/client/core/arrow/ArrayConverter.java | 2 +- .../net/snowflake/client/core/arrow/MapConverter.java | 2 +- .../net/snowflake/client/core/arrow/StructConverter.java | 2 +- ...StructuredTypesGetStringArrowJsonCompatibilityIT.java | 9 --------- .../structuredtypes/StructuredTypesGetStringBaseIT.java | 6 ------ 6 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index efbc304a2..91ab33975 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -572,8 +572,12 @@ public Object getObject(int columnIndex) throws SFException { converter.setTreatNTZAsUTC(treatNTZAsUTC); converter.setUseSessionTimezone(useSessionTimezone); converter.setSessionTimeZone(sessionTimeZone); - - return converter.toString(index); + Object obj = converter.toObject(index); + boolean isStructuredType = resultSetMetaData.isStructuredTypeColumn(columnIndex); + if (type == Types.STRUCT && isStructuredType && converter instanceof VarCharConverter) { + return createJsonSqlInput(columnIndex, obj); + } + return obj; } private Object createJsonSqlInput(int columnIndex, Object obj) throws SFException { diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 13ebce18f..2ac783a8b 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -19,7 +19,7 @@ public ArrayConverter(ListVector valueVector, int vectorIndex, DataConversionCon @Override public Object toObject(int index) throws SFException { - return toString(index); + return isNull(index)? null : toString(index); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index c869a6205..eca7e22ce 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -19,7 +19,7 @@ public MapConverter(MapVector valueVector, int columnIndex, DataConversionContex @Override public Object toObject(int index) throws SFException { - return toString(index); + return isNull(index)? null : toString(index); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index db9554660..020331840 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -21,7 +21,7 @@ public StructConverter(StructVector vector, int columnIndex, DataConversionConte @Override public Object toObject(int index) throws SFException { - return toString(index); + return isNull(index)? null : toString(index); } @Override diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index 66ccb0096..7ef9b2588 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -71,15 +71,6 @@ public void testRunAsGetObject() throws SQLException { (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); } - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testRunAsGetBytes() throws SQLException { - withFirstRow( - connections.get(queryResultFormat), - selectSql, - (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); - } - @Parameterized.Parameters(name = "format={0},sql={1}") public static Collection data() { Map samples = new LinkedHashMap<>(); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 9c8b9c29a..32b034846 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -51,12 +51,6 @@ protected static Connection initConnection(ResultSetFormatType queryResultFormat return conn; } - protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) - throws SQLException { - String result = new String(resultSet.getBytes(1), StandardCharsets.UTF_8); - TestUtil.assertEqualsIgnoringWhitespace(expected, result); - } - protected void assertGetStringIsCompatible(ResultSet resultSet, String expected) throws SQLException { String result = resultSet.getString(1); From c807e367705b999ff0b6d9d73a05b70457fb4344 Mon Sep 17 00:00:00 2001 From: Przemyslaw Motacki Date: Wed, 30 Oct 2024 10:47:23 +0100 Subject: [PATCH 05/29] SNOW-1374896 Returning json string from method getObject when resultFormat ARROW is set --- .../client/core/SFArrowResultSet.java | 43 ++++++++++++------- .../client/core/SFBaseResultSet.java | 15 +++++++ .../client/core/SFJsonResultSet.java | 3 +- .../net/snowflake/client/core/SfSqlArray.java | 14 ++++++ .../client/core/arrow/ArrayConverter.java | 2 +- .../client/core/arrow/MapConverter.java | 13 +++++- .../client/core/arrow/StructConverter.java | 12 ++++-- .../client/core/arrow/StructObject.java | 22 ++++++++++ .../core/arrow/VectorTypeConverter.java | 3 +- .../ArrowStringRepresentationBuilderBase.java | 3 ++ .../client/jdbc/SnowflakeBaseResultSet.java | 34 ++++++++++++--- .../client/jdbc/SnowflakeResultSetV1.java | 6 +++ .../ResultSetStructuredTypesLatestIT.java | 33 ++++++-------- .../StructuredTypesGetStringBaseIT.java | 1 - 14 files changed, 153 insertions(+), 51 deletions(-) create mode 100644 src/main/java/net/snowflake/client/core/arrow/StructObject.java diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 91ab33975..b76ba2f82 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import net.snowflake.client.core.arrow.ArrayConverter; import net.snowflake.client.core.arrow.ArrowVectorConverter; +import net.snowflake.client.core.arrow.StructObject; import net.snowflake.client.core.arrow.VarCharConverter; import net.snowflake.client.core.arrow.VectorTypeConverter; import net.snowflake.client.core.json.Converters; @@ -575,7 +576,10 @@ public Object getObject(int columnIndex) throws SFException { Object obj = converter.toObject(index); boolean isStructuredType = resultSetMetaData.isStructuredTypeColumn(columnIndex); if (type == Types.STRUCT && isStructuredType && converter instanceof VarCharConverter) { - return createJsonSqlInput(columnIndex, obj); + if (obj == null) { + return null; + } + return new StructObject((String) obj, createJsonSqlInput(columnIndex, obj)); } return obj; } @@ -599,15 +603,6 @@ private Object createJsonSqlInput(int columnIndex, Object obj) throws SFExceptio } } - private Object createArrowSqlInput(int columnIndex, Map input) - throws SFException { - if (input == null) { - return null; - } - return new ArrowSqlInput( - input, session, converters, resultSetMetaData.getColumnFields(columnIndex)); - } - @Override public Array getArray(int columnIndex) throws SFException { ArrowVectorConverter converter = currentChunkIterator.getCurrentConverter(columnIndex - 1); @@ -619,16 +614,17 @@ public Array getArray(int columnIndex) throws SFException { } if (converter instanceof VarCharConverter) { return getJsonArray((String) obj, columnIndex); - } else if (converter instanceof ArrayConverter) { - return getArrowArray((List) obj, columnIndex); - } else if (converter instanceof VectorTypeConverter) { - return getArrowArray((List) obj, columnIndex); + } else if (converter instanceof ArrayConverter || converter instanceof VectorTypeConverter) { + StructObject structObject = (StructObject) obj; + return getArrowArray( + structObject.getStringJson(), (List) structObject.getObject(), columnIndex); } else { throw new SFException(queryId, ErrorCode.INVALID_STRUCT_DATA); } } - private SfSqlArray getArrowArray(List elements, int columnIndex) throws SFException { + private SfSqlArray getArrowArray(String text, List elements, int columnIndex) + throws SFException { try { List fieldMetadataList = resultSetMetaData.getColumnFields(columnIndex); if (fieldMetadataList.size() != 1) { @@ -645,26 +641,31 @@ private SfSqlArray getArrowArray(List elements, int columnIndex) throws switch (columnType) { case Types.INTEGER: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.integerConverter(columnType)) .toArray(Integer[]::new)); case Types.SMALLINT: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.smallIntConverter(columnType)) .toArray(Short[]::new)); case Types.TINYINT: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.tinyIntConverter(columnType)) .toArray(Byte[]::new)); case Types.BIGINT: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.bigIntConverter(columnType)).toArray(Long[]::new)); case Types.DECIMAL: case Types.NUMERIC: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.bigDecimalConverter(columnType)) .toArray(BigDecimal[]::new)); @@ -672,35 +673,42 @@ private SfSqlArray getArrowArray(List elements, int columnIndex) throws case Types.VARCHAR: case Types.LONGNVARCHAR: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.varcharConverter(columnType, columnSubType, scale)) .toArray(String[]::new)); case Types.BINARY: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.bytesConverter(columnType, scale)) .toArray(Byte[][]::new)); case Types.FLOAT: case Types.REAL: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.floatConverter(columnType)).toArray(Float[]::new)); case Types.DOUBLE: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.doubleConverter(columnType)) .toArray(Double[]::new)); case Types.DATE: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.dateFromIntConverter(sessionTimeZone)) .toArray(Date[]::new)); case Types.TIME: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.timeFromIntConverter(scale)).toArray(Time[]::new)); case Types.TIMESTAMP: return new SfSqlArray( + text, columnSubType, mapAndConvert( elements, @@ -709,13 +717,16 @@ private SfSqlArray getArrowArray(List elements, int columnIndex) throws .toArray(Timestamp[]::new)); case Types.BOOLEAN: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, converters.booleanConverter(columnType)) .toArray(Boolean[]::new)); case Types.STRUCT: - return new SfSqlArray(columnSubType, mapAndConvert(elements, e -> e).toArray(Map[]::new)); + return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, e -> e).toArray(Map[]::new)); case Types.ARRAY: return new SfSqlArray( + text, columnSubType, mapAndConvert(elements, e -> ((List) e).stream().toArray(Map[]::new)) .toArray(Map[][]::new)); diff --git a/src/main/java/net/snowflake/client/core/SFBaseResultSet.java b/src/main/java/net/snowflake/client/core/SFBaseResultSet.java index 71e56a515..e13ceb2fb 100644 --- a/src/main/java/net/snowflake/client/core/SFBaseResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFBaseResultSet.java @@ -294,27 +294,32 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio switch (columnType) { case Types.INTEGER: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().integerConverter(columnType)) .toArray(Integer[]::new)); case Types.SMALLINT: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().smallIntConverter(columnType)) .toArray(Short[]::new)); case Types.TINYINT: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().tinyIntConverter(columnType)) .toArray(Byte[]::new)); case Types.BIGINT: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().bigIntConverter(columnType)) .toArray(Long[]::new)); case Types.DECIMAL: case Types.NUMERIC: return new SfSqlArray( + obj, columnSubType, convertToFixedArray( getStream(nodeElements, getConverters().bigDecimalConverter(columnType)))); @@ -322,6 +327,7 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio case Types.VARCHAR: case Types.LONGNVARCHAR: return new SfSqlArray( + obj, columnSubType, getStream( nodeElements, @@ -329,32 +335,38 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio .toArray(String[]::new)); case Types.BINARY: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().bytesConverter(columnType, scale)) .toArray(Byte[][]::new)); case Types.FLOAT: case Types.REAL: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().floatConverter(columnType)) .toArray(Float[]::new)); case Types.DOUBLE: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().doubleConverter(columnType)) .toArray(Double[]::new)); case Types.DATE: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().dateStringConverter(session)) .toArray(Date[]::new)); case Types.TIME: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().timeFromStringConverter(session)) .toArray(Time[]::new)); case Types.TIMESTAMP: return new SfSqlArray( + obj, columnSubType, getStream( nodeElements, @@ -364,16 +376,19 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio .toArray(Timestamp[]::new)); case Types.BOOLEAN: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().booleanConverter(columnType)) .toArray(Boolean[]::new)); case Types.STRUCT: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().structConverter(OBJECT_MAPPER)) .toArray(Map[]::new)); case Types.ARRAY: return new SfSqlArray( + obj, columnSubType, getStream(nodeElements, getConverters().arrayConverter(OBJECT_MAPPER)) .toArray(Map[][]::new)); diff --git a/src/main/java/net/snowflake/client/core/SFJsonResultSet.java b/src/main/java/net/snowflake/client/core/SFJsonResultSet.java index 1011870df..ca2c9646e 100644 --- a/src/main/java/net/snowflake/client/core/SFJsonResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFJsonResultSet.java @@ -15,6 +15,7 @@ import java.sql.Types; import java.util.List; import java.util.TimeZone; +import net.snowflake.client.core.arrow.StructObject; import net.snowflake.client.core.json.Converters; import net.snowflake.client.jdbc.ErrorCode; import net.snowflake.client.jdbc.FieldMetadata; @@ -87,7 +88,7 @@ public Object getObject(int columnIndex) throws SFException { case Types.STRUCT: if (resultSetMetaData.isStructuredTypeColumn(columnIndex)) { - return getSqlInput((String) obj, columnIndex); + return new StructObject((String) obj, getSqlInput((String) obj, columnIndex)); } else { throw new SFException(ErrorCode.FEATURE_UNSUPPORTED, "data type: " + type); } diff --git a/src/main/java/net/snowflake/client/core/SfSqlArray.java b/src/main/java/net/snowflake/client/core/SfSqlArray.java index 70682b4f4..5d3675cf0 100644 --- a/src/main/java/net/snowflake/client/core/SfSqlArray.java +++ b/src/main/java/net/snowflake/client/core/SfSqlArray.java @@ -16,9 +16,16 @@ @SnowflakeJdbcInternalApi public class SfSqlArray implements Array { + private String text; private int baseType; private Object elements; + public SfSqlArray(String text, int baseType, Object elements) { + this.text = text; + this.baseType = baseType; + this.elements = elements; + } + public SfSqlArray(int baseType, Object elements) { this.baseType = baseType; this.elements = elements; @@ -82,6 +89,13 @@ public ResultSet getResultSet(long index, int count, Map> map) public void free() throws SQLException {} public String getJsonString() throws SQLException { + if (text == null) { + text = buildJsonStringFromElements(elements); + } + return text; + } + + private static String buildJsonStringFromElements(Object elements) throws SQLException { try { return SnowflakeUtil.mapJson(elements); } catch (JsonProcessingException e) { diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 2ac783a8b..a61ca379f 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -19,7 +19,7 @@ public ArrayConverter(ListVector valueVector, int vectorIndex, DataConversionCon @Override public Object toObject(int index) throws SFException { - return isNull(index)? null : toString(index); + return isNull(index) ? null : new StructObject(toString(index), vector.getObject(index)); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index eca7e22ce..869d5e200 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -1,5 +1,8 @@ package net.snowflake.client.core.arrow; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.arrow.tostringhelpers.ArrowObjectStringRepresentationBuilder; @@ -7,6 +10,7 @@ import net.snowflake.client.jdbc.SnowflakeType; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.util.JsonStringHashMap; public class MapConverter extends AbstractArrowVectorConverter { @@ -19,7 +23,14 @@ public MapConverter(MapVector valueVector, int columnIndex, DataConversionContex @Override public Object toObject(int index) throws SFException { - return isNull(index)? null : toString(index); + List> entriesList = + (List>) vector.getObject(index); + Map map = + entriesList.stream() + .collect( + Collectors.toMap( + entry -> entry.get("key").toString(), entry -> entry.get("value"))); + return isNull(index) ? null : new StructObject(toString(index), map); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 020331840..13c446ca4 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -21,7 +21,7 @@ public StructConverter(StructVector vector, int columnIndex, DataConversionConte @Override public Object toObject(int index) throws SFException { - return isNull(index)? null : toString(index); + return isNull(index) ? null : new StructObject(toString(index), structVector.getObject(index)); } @Override @@ -32,9 +32,13 @@ public String toString(int index) throws SFException { SnowflakeType logicalType = ArrowVectorConverterUtil.getSnowflakeTypeFromFieldMetadata(fieldVector.getField()); try { - ArrowVectorConverter converter = - ArrowVectorConverterUtil.initConverter(fieldVector, context, columnIndex); - builder.appendKeyValue(childName, converter.toString(index), logicalType); + if (fieldVector.isNull(index)) { + builder.appendKeyValue(childName, null, logicalType); + } else { + ArrowVectorConverter converter = + ArrowVectorConverterUtil.initConverter(fieldVector, context, columnIndex); + builder.appendKeyValue(childName, converter.toString(index), logicalType); + } } catch (SnowflakeSQLException e) { return structVector.getObject(index).toString(); } diff --git a/src/main/java/net/snowflake/client/core/arrow/StructObject.java b/src/main/java/net/snowflake/client/core/arrow/StructObject.java new file mode 100644 index 000000000..446d71436 --- /dev/null +++ b/src/main/java/net/snowflake/client/core/arrow/StructObject.java @@ -0,0 +1,22 @@ +package net.snowflake.client.core.arrow; + +import net.snowflake.client.core.SnowflakeJdbcInternalApi; + +@SnowflakeJdbcInternalApi +public class StructObject { + private final String stringJson; + private final Object object; + + public StructObject(String stringJson, Object object) { + this.stringJson = stringJson; + this.object = object; + } + + public String getStringJson() { + return stringJson; + } + + public Object getObject() { + return object; + } +} diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index ae7a492a0..a6d414726 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -18,7 +18,8 @@ public VectorTypeConverter( @Override public Object toObject(int index) throws SFException { - return vector.getObject(index); + Object object = vector.getObject(index); + return new StructObject(object.toString(), object); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java index cc25bb7e0..45df80cf3 100644 --- a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java +++ b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java @@ -44,6 +44,9 @@ private boolean shouldQuoteValue(SnowflakeType type) { } protected String quoteIfNeeded(String string, SnowflakeType type) { + if (string == null) { + return null; + } // Turn Boolean string representations lowercase to make the output JSON-compatible // this should be changed on the converter level, but it would be a breaking change thus // for now only structured types will be valid JSONs while in NATIVE ARROW mode diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java index d9149412e..2878ec2c9 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java @@ -45,6 +45,7 @@ import net.snowflake.client.core.SFBaseResultSet; import net.snowflake.client.core.SFBaseSession; import net.snowflake.client.core.SFException; +import net.snowflake.client.core.arrow.StructObject; import net.snowflake.client.core.structs.SQLDataCreationHelper; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; @@ -1378,7 +1379,10 @@ public T getObject(int columnIndex, Class type) throws SQLException { if (SQLData.class.isAssignableFrom(type)) { SQLInput sqlInput = SnowflakeUtil.mapSFExceptionToSQLException( - () -> (SQLInput) sfBaseResultSet.getObject(columnIndex)); + () -> { + StructObject structObject = (StructObject) sfBaseResultSet.getObject(columnIndex); + return (SQLInput) createJsonSqlInput(columnIndex, structObject); + }); if (sqlInput == null) { return null; } else { @@ -1614,13 +1618,15 @@ public Map getMap(int columnIndex, Class type) throws SQLExcep int columnType = ColumnTypeHelper.getColumnType(valueFieldMetadata.getType(), session); int scale = valueFieldMetadata.getScale(); TimeZone tz = sfBaseResultSet.getSessionTimeZone(); - Object object = - SnowflakeUtil.mapSFExceptionToSQLException(() -> sfBaseResultSet.getObject(columnIndex)); + StructObject object = + (StructObject) + SnowflakeUtil.mapSFExceptionToSQLException( + () -> sfBaseResultSet.getObject(columnIndex)); if (object == null) { return null; } Map map = - mapSFExceptionToSQLException(() -> prepareMapWithValues(object, type)); + mapSFExceptionToSQLException(() -> prepareMapWithValues(object.getObject(), type)); Map resultMap = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { if (SQLData.class.isAssignableFrom(type)) { @@ -1628,7 +1634,7 @@ public Map getMap(int columnIndex, Class type) throws SQLExcep SQLInput sqlInput = sfBaseResultSet.createSqlInputForColumn( entry.getValue(), - object.getClass(), + object.getObject().getClass(), columnIndex, session, valueFieldMetadata.getFields()); @@ -1807,4 +1813,22 @@ private Map prepareMapWithValues(Object object, Class typ throw new SFException(ErrorCode.INVALID_STRUCT_DATA, "Object couldn't be converted to map"); } } + + private Object createJsonSqlInput(int columnIndex, StructObject obj) throws SFException { + try { + if (obj == null) { + return null; + } + JsonNode jsonNode = OBJECT_MAPPER.readTree(obj.getStringJson()); + return new JsonSqlInput( + obj.getStringJson(), + jsonNode, + session, + sfBaseResultSet.getConverters(), + sfBaseResultSet.getMetaData().getColumnFields(columnIndex), + sfBaseResultSet.getSessionTimeZone()); + } catch (JsonProcessingException e) { + throw new SFException(sfBaseResultSet.getQueryId(), e, ErrorCode.INVALID_STRUCT_DATA); + } + } } diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java index 49c8c8546..fefa508e5 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java @@ -33,6 +33,8 @@ import net.snowflake.client.core.QueryStatus; import net.snowflake.client.core.SFBaseResultSet; import net.snowflake.client.core.SFException; +import net.snowflake.client.core.SfSqlArray; +import net.snowflake.client.core.arrow.StructObject; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; @@ -274,6 +276,10 @@ public Object getObject(int columnIndex) throws SQLException { return null; } else if (object instanceof JsonSqlInput) { return ((JsonSqlInput) object).getText(); + } else if (object instanceof StructObject) { + return ((StructObject) object).getStringJson(); + } else if (object instanceof SfSqlArray) { + return ((SfSqlArray) object).getJsonString(); } else if (object instanceof ArrowSqlInput) { throw new SQLException( "Arrow native struct couldn't be converted to String. To map to SqlData the method getObject(int columnIndex, Class type) should be used"); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java index 2857634f8..77fdf859a 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java @@ -6,7 +6,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.math.BigDecimal; @@ -193,7 +192,6 @@ public void testMapStructAllTypes() throws SQLException { @Test @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testReturnStructAsStringIfTypeWasNotIndicated() throws SQLException { - Assume.assumeTrue(queryResultFormat != ResultSetFormatType.NATIVE_ARROW); try (Connection connection = init(); Statement statement = connection.createStatement()) { statement.execute( @@ -210,7 +208,7 @@ public void testReturnStructAsStringIfTypeWasNotIndicated() throws SQLException try (ResultSet resultSet = statement.executeQuery(AllTypesClass.ALL_TYPES_QUERY); ) { resultSet.next(); String object = (String) resultSet.getObject(1); - String expected = + String expectedJson = "{\n" + " \"string\": \"a\",\n" + " \"b\": 1,\n" @@ -232,28 +230,21 @@ public void testReturnStructAsStringIfTypeWasNotIndicated() throws SQLException + " \"intValue\": 2\n" + " }\n" + "}"; - assertEquals(expected, object); + String expectedJsonFromArrow = + "{\"string\": \"a\",\"b\": 1,\"s\": 2,\"i\": 3,\"l\": 4,\"f\": 1.1,\"d\": 2.2,\"bd\": 3.3," + + "\"bool\": true,\"timestamp_ltz\": \"2021-12-22 09:43:44.000 +0100\",\"timestamp_ntz\": \"2021-12-23 09:44:44.000\"," + + "\"timestamp_tz\": \"2021-12-24 09:45:45.000 +0800\",\"date\": \"2023-12-24\",\"time\": \"12:34:56\",\"binary\": \"616263\"," + + "\"simpleClass\": {\"string\": \"b\",\"intValue\": 2}}"; + + if (queryResultFormat == ResultSetFormatType.NATIVE_ARROW) { + assertEquals(expectedJsonFromArrow, object); + } else { + assertEquals(expectedJson, object); + } } } } - @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) - public void testThrowingGettingObjectIfTypeWasNotIndicatedAndFormatNativeArrow() - throws SQLException { - Assume.assumeTrue(queryResultFormat == ResultSetFormatType.NATIVE_ARROW); - withFirstRow( - "select {'string':'a'}::OBJECT(string VARCHAR)", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1)); - }); - withFirstRow( - "select {'x':{'string':'one'},'y':{'string':'two'},'z':{'string':'three'}}::MAP(VARCHAR, OBJECT(string VARCHAR));", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1, Map.class)); - }); - } - @Test @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testReturnAsArrayOfSqlData() throws SQLException { diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 32b034846..ffaca45c8 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -2,7 +2,6 @@ import static org.junit.Assert.assertTrue; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; From 4a74df7263c5370e37284a8d9c2eb673e57f1f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Thu, 14 Nov 2024 15:54:34 +0100 Subject: [PATCH 06/29] address review comments --- .../client/core/SFArrowResultSet.java | 8 +++---- .../client/core/SFJsonResultSet.java | 4 ++-- .../client/core/arrow/ArrayConverter.java | 2 +- .../client/core/arrow/MapConverter.java | 8 +++++-- .../client/core/arrow/StructConverter.java | 2 +- ...ctObject.java => StructObjectWrapper.java} | 12 +++++----- .../core/arrow/VectorTypeConverter.java | 5 ++++- .../client/jdbc/SnowflakeBaseResultSet.java | 22 +++++++++---------- .../client/jdbc/SnowflakeResultSetV1.java | 6 ++--- ...ypesGetStringArrowJsonCompatibilityIT.java | 2 -- 10 files changed, 38 insertions(+), 33 deletions(-) rename src/main/java/net/snowflake/client/core/arrow/{StructObject.java => StructObjectWrapper.java} (52%) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index f393efd55..9edffe66b 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -26,7 +26,7 @@ import java.util.stream.Stream; import net.snowflake.client.core.arrow.ArrayConverter; import net.snowflake.client.core.arrow.ArrowVectorConverter; -import net.snowflake.client.core.arrow.StructObject; +import net.snowflake.client.core.arrow.StructObjectWrapper; import net.snowflake.client.core.arrow.VarCharConverter; import net.snowflake.client.core.arrow.VectorTypeConverter; import net.snowflake.client.core.json.Converters; @@ -580,7 +580,7 @@ public Object getObject(int columnIndex) throws SFException { if (obj == null) { return null; } - return new StructObject((String) obj, createJsonSqlInput(columnIndex, obj)); + return new StructObjectWrapper((String) obj, createJsonSqlInput(columnIndex, obj)); } return obj; } @@ -616,9 +616,9 @@ public Array getArray(int columnIndex) throws SFException { if (converter instanceof VarCharConverter) { return getJsonArray((String) obj, columnIndex); } else if (converter instanceof ArrayConverter || converter instanceof VectorTypeConverter) { - StructObject structObject = (StructObject) obj; + StructObjectWrapper structObjectWrapper = (StructObjectWrapper) obj; return getArrowArray( - structObject.getStringJson(), (List) structObject.getObject(), columnIndex); + structObjectWrapper.getJsonString(), (List) structObjectWrapper.getObject(), columnIndex); } else { throw new SFException(queryId, ErrorCode.INVALID_STRUCT_DATA); } diff --git a/src/main/java/net/snowflake/client/core/SFJsonResultSet.java b/src/main/java/net/snowflake/client/core/SFJsonResultSet.java index dffd5cfa6..04e8d3fba 100644 --- a/src/main/java/net/snowflake/client/core/SFJsonResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFJsonResultSet.java @@ -15,7 +15,7 @@ import java.sql.Types; import java.util.List; import java.util.TimeZone; -import net.snowflake.client.core.arrow.StructObject; +import net.snowflake.client.core.arrow.StructObjectWrapper; import net.snowflake.client.core.json.Converters; import net.snowflake.client.jdbc.ErrorCode; import net.snowflake.client.jdbc.FieldMetadata; @@ -88,7 +88,7 @@ public Object getObject(int columnIndex) throws SFException { case Types.STRUCT: if (resultSetMetaData.isStructuredTypeColumn(columnIndex)) { - return new StructObject((String) obj, getSqlInput((String) obj, columnIndex)); + return new StructObjectWrapper((String) obj, getSqlInput((String) obj, columnIndex)); } else { throw new SFException(ErrorCode.FEATURE_UNSUPPORTED, "data type: " + type); } diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index dd641b21a..c98ce9a47 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -25,7 +25,7 @@ public ArrayConverter(ListVector valueVector, int vectorIndex, DataConversionCon @Override public Object toObject(int index) throws SFException { - return isNull(index) ? null : new StructObject(toString(index), vector.getObject(index)); + return isNull(index) ? null : new StructObjectWrapper(toString(index), vector.getObject(index)); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 21be9ab53..3779896e8 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -29,14 +29,18 @@ public MapConverter(MapVector valueVector, int columnIndex, DataConversionContex @Override public Object toObject(int index) throws SFException { + if (isNull(index)) { + return null; + } + List> entriesList = (List>) vector.getObject(index); - Map map = + Map map = entriesList.stream() .collect( Collectors.toMap( entry -> entry.get("key").toString(), entry -> entry.get("value"))); - return isNull(index) ? null : new StructObject(toString(index), map); + return new StructObjectWrapper(toString(index), map); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 13c446ca4..d246045cb 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -21,7 +21,7 @@ public StructConverter(StructVector vector, int columnIndex, DataConversionConte @Override public Object toObject(int index) throws SFException { - return isNull(index) ? null : new StructObject(toString(index), structVector.getObject(index)); + return isNull(index) ? null : new StructObjectWrapper(toString(index), structVector.getObject(index)); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructObject.java b/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java similarity index 52% rename from src/main/java/net/snowflake/client/core/arrow/StructObject.java rename to src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java index 446d71436..b69b4645e 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructObject.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java @@ -3,17 +3,17 @@ import net.snowflake.client.core.SnowflakeJdbcInternalApi; @SnowflakeJdbcInternalApi -public class StructObject { - private final String stringJson; +public class StructObjectWrapper { + private final String jsonString; private final Object object; - public StructObject(String stringJson, Object object) { - this.stringJson = stringJson; + public StructObjectWrapper(String jsonString, Object object) { + this.jsonString = jsonString; this.object = object; } - public String getStringJson() { - return stringJson; + public String getJsonString() { + return jsonString; } public Object getObject() { diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index 6199c0b8c..80af9dec2 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -24,8 +24,11 @@ public VectorTypeConverter( @Override public Object toObject(int index) throws SFException { + if (isNull(index)) { + return null; + } Object object = vector.getObject(index); - return new StructObject(object.toString(), object); + return new StructObjectWrapper(object.toString(), object); } @Override diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java index c2ed2bff6..eb07dd8c9 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java @@ -45,7 +45,7 @@ import net.snowflake.client.core.SFBaseResultSet; import net.snowflake.client.core.SFBaseSession; import net.snowflake.client.core.SFException; -import net.snowflake.client.core.arrow.StructObject; +import net.snowflake.client.core.arrow.StructObjectWrapper; import net.snowflake.client.core.structs.SQLDataCreationHelper; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; @@ -1397,8 +1397,8 @@ public T getObject(int columnIndex, Class type) throws SQLException { SQLInput sqlInput = SnowflakeUtil.mapSFExceptionToSQLException( () -> { - StructObject structObject = (StructObject) sfBaseResultSet.getObject(columnIndex); - return (SQLInput) createJsonSqlInput(columnIndex, structObject); + StructObjectWrapper structObjectWrapper = (StructObjectWrapper) sfBaseResultSet.getObject(columnIndex); + return (SQLInput) createJsonSqlInput(columnIndex, structObjectWrapper); }); if (sqlInput == null) { return null; @@ -1635,15 +1635,15 @@ public Map getMap(int columnIndex, Class type) throws SQLExcep int columnType = ColumnTypeHelper.getColumnType(valueFieldMetadata.getType(), session); int scale = valueFieldMetadata.getScale(); TimeZone tz = sfBaseResultSet.getSessionTimeZone(); - StructObject object = - (StructObject) + StructObjectWrapper structObjectWrapper = + (StructObjectWrapper) SnowflakeUtil.mapSFExceptionToSQLException( () -> sfBaseResultSet.getObject(columnIndex)); - if (object == null) { + if (structObjectWrapper == null) { return null; } Map map = - mapSFExceptionToSQLException(() -> prepareMapWithValues(object.getObject(), type)); + mapSFExceptionToSQLException(() -> prepareMapWithValues(structObjectWrapper.getObject(), type)); Map resultMap = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { if (SQLData.class.isAssignableFrom(type)) { @@ -1651,7 +1651,7 @@ public Map getMap(int columnIndex, Class type) throws SQLExcep SQLInput sqlInput = sfBaseResultSet.createSqlInputForColumn( entry.getValue(), - object.getObject().getClass(), + structObjectWrapper.getObject().getClass(), columnIndex, session, valueFieldMetadata.getFields()); @@ -1831,14 +1831,14 @@ private Map prepareMapWithValues(Object object, Class typ } } - private Object createJsonSqlInput(int columnIndex, StructObject obj) throws SFException { + private Object createJsonSqlInput(int columnIndex, StructObjectWrapper obj) throws SFException { try { if (obj == null) { return null; } - JsonNode jsonNode = OBJECT_MAPPER.readTree(obj.getStringJson()); + JsonNode jsonNode = OBJECT_MAPPER.readTree(obj.getJsonString()); return new JsonSqlInput( - obj.getStringJson(), + obj.getJsonString(), jsonNode, session, sfBaseResultSet.getConverters(), diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java index 3575e0a38..dfe8cbad6 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java @@ -34,7 +34,7 @@ import net.snowflake.client.core.SFBaseResultSet; import net.snowflake.client.core.SFException; import net.snowflake.client.core.SfSqlArray; -import net.snowflake.client.core.arrow.StructObject; +import net.snowflake.client.core.arrow.StructObjectWrapper; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; @@ -276,8 +276,8 @@ public Object getObject(int columnIndex) throws SQLException { return null; } else if (object instanceof JsonSqlInput) { return ((JsonSqlInput) object).getText(); - } else if (object instanceof StructObject) { - return ((StructObject) object).getStringJson(); + } else if (object instanceof StructObjectWrapper) { + return ((StructObjectWrapper) object).getJsonString(); } else if (object instanceof SfSqlArray) { return ((SfSqlArray) object).getJsonString(); } else if (object instanceof ArrowSqlInput) { diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index 7ef9b2588..ffc19aeef 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -54,7 +54,6 @@ public static void closeConnections() throws SQLException { } @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testRunAsGetString() throws SQLException { withFirstRow( connections.get(queryResultFormat), @@ -63,7 +62,6 @@ public void testRunAsGetString() throws SQLException { } @Test - @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGithubAction.class) public void testRunAsGetObject() throws SQLException { withFirstRow( connections.get(queryResultFormat), From 219d3c54970cef9156fc949ff2806d31839e72b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Thu, 14 Nov 2024 16:03:17 +0100 Subject: [PATCH 07/29] fix formatting --- .../java/net/snowflake/client/core/SFArrowResultSet.java | 4 +++- .../net/snowflake/client/core/arrow/StructConverter.java | 4 +++- .../net/snowflake/client/jdbc/SnowflakeBaseResultSet.java | 6 ++++-- .../StructuredTypesGetStringArrowJsonCompatibilityIT.java | 2 -- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 9edffe66b..63267302d 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -618,7 +618,9 @@ public Array getArray(int columnIndex) throws SFException { } else if (converter instanceof ArrayConverter || converter instanceof VectorTypeConverter) { StructObjectWrapper structObjectWrapper = (StructObjectWrapper) obj; return getArrowArray( - structObjectWrapper.getJsonString(), (List) structObjectWrapper.getObject(), columnIndex); + structObjectWrapper.getJsonString(), + (List) structObjectWrapper.getObject(), + columnIndex); } else { throw new SFException(queryId, ErrorCode.INVALID_STRUCT_DATA); } diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index d246045cb..721f0a4ff 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -21,7 +21,9 @@ public StructConverter(StructVector vector, int columnIndex, DataConversionConte @Override public Object toObject(int index) throws SFException { - return isNull(index) ? null : new StructObjectWrapper(toString(index), structVector.getObject(index)); + return isNull(index) + ? null + : new StructObjectWrapper(toString(index), structVector.getObject(index)); } @Override diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java index eb07dd8c9..633083391 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeBaseResultSet.java @@ -1397,7 +1397,8 @@ public T getObject(int columnIndex, Class type) throws SQLException { SQLInput sqlInput = SnowflakeUtil.mapSFExceptionToSQLException( () -> { - StructObjectWrapper structObjectWrapper = (StructObjectWrapper) sfBaseResultSet.getObject(columnIndex); + StructObjectWrapper structObjectWrapper = + (StructObjectWrapper) sfBaseResultSet.getObject(columnIndex); return (SQLInput) createJsonSqlInput(columnIndex, structObjectWrapper); }); if (sqlInput == null) { @@ -1643,7 +1644,8 @@ public Map getMap(int columnIndex, Class type) throws SQLExcep return null; } Map map = - mapSFExceptionToSQLException(() -> prepareMapWithValues(structObjectWrapper.getObject(), type)); + mapSFExceptionToSQLException( + () -> prepareMapWithValues(structObjectWrapper.getObject(), type)); Map resultMap = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { if (SQLData.class.isAssignableFrom(type)) { diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index ffc19aeef..a2074e66a 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -7,8 +7,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -import net.snowflake.client.ConditionalIgnoreRule; -import net.snowflake.client.RunningOnGithubAction; import net.snowflake.client.category.TestCategoryResultSet; import net.snowflake.client.jdbc.ResultSetFormatType; import org.junit.AfterClass; From d1ca63093a2d740cc0b3cbbef222a0be7b132325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Thu, 14 Nov 2024 16:18:32 +0100 Subject: [PATCH 08/29] remove variant from quotable type in ArrowStringRepresentationBuilder, add variant test case --- .../tostringhelpers/ArrowStringRepresentationBuilderBase.java | 1 - .../StructuredTypesGetStringArrowJsonCompatibilityIT.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java index 45df80cf3..f60daa0cc 100644 --- a/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java +++ b/src/main/java/net/snowflake/client/core/arrow/tostringhelpers/ArrowStringRepresentationBuilderBase.java @@ -21,7 +21,6 @@ public abstract class ArrowStringRepresentationBuilderBase { quotableTypes.add(SnowflakeType.ANY); quotableTypes.add(SnowflakeType.CHAR); quotableTypes.add(SnowflakeType.TEXT); - quotableTypes.add(SnowflakeType.VARIANT); quotableTypes.add(SnowflakeType.BINARY); quotableTypes.add(SnowflakeType.DATE); quotableTypes.add(SnowflakeType.TIME); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java index a2074e66a..5f7a46ff2 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java @@ -131,6 +131,7 @@ public static Collection data() { "{\"binary\":\"616263\"}"); samples.put("select [1,2,3]::VECTOR(INT, 3)", "[1,2,3]"); samples.put("select ['a','b','c']::ARRAY(varchar)", "[\"a\",\"b\",\"c\"]"); + samples.put("select ['a','b','c']::ARRAY(variant)", "[\"a\",\"b\",\"c\"]"); Collection parameters = new ArrayList<>(); for (ResultSetFormatType resultSetFormatType : ResultSetFormatType.values()) { From 230e9354d7a491c9eecfdd87c6c4d5fbd00744d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 15 Nov 2024 16:59:41 +0100 Subject: [PATCH 09/29] add toBytes returning toString.toBytes, make compatibility tests latest --- .../snowflake/client/core/arrow/ArrayConverter.java | 5 +++++ .../snowflake/client/core/arrow/MapConverter.java | 5 +++++ .../snowflake/client/core/arrow/StructConverter.java | 5 +++++ .../client/core/arrow/VectorTypeConverter.java | 5 +++++ .../snowflake/client/core/json/BytesConverter.java | 3 +++ ...ructuredTypesArrowJsonCompatibilityLatestIT.java} | 12 ++++++++++-- .../StructuredTypesGetStringBaseIT.java | 11 +++++++++++ 7 files changed, 44 insertions(+), 2 deletions(-) rename src/test/java/net/snowflake/client/jdbc/structuredtypes/{StructuredTypesGetStringArrowJsonCompatibilityIT.java => StructuredTypesArrowJsonCompatibilityLatestIT.java} (94%) diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index c98ce9a47..96b683151 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -28,6 +28,11 @@ public Object toObject(int index) throws SFException { return isNull(index) ? null : new StructObjectWrapper(toString(index), vector.getObject(index)); } + @Override + public byte[] toBytes(int index) throws SFException { + return isNull(index) ? null : toString(index).getBytes(); + } + @Override public String toString(int index) throws SFException { FieldVector vectorUnpacked = vector.getChildrenFromFields().get(0); diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 3779896e8..e4e148ac5 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -43,6 +43,11 @@ public Object toObject(int index) throws SFException { return new StructObjectWrapper(toString(index), map); } + @Override + public byte[] toBytes(int index) throws SFException { + return isNull(index) ? null : toString(index).getBytes(); + } + @Override public String toString(int index) throws SFException { ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 721f0a4ff..ab7d20382 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -26,6 +26,11 @@ public Object toObject(int index) throws SFException { : new StructObjectWrapper(toString(index), structVector.getObject(index)); } + @Override + public byte[] toBytes(int index) throws SFException { + return isNull(index) ? null : toString(index).getBytes(); + } + @Override public String toString(int index) throws SFException { ArrowObjectStringRepresentationBuilder builder = new ArrowObjectStringRepresentationBuilder(); diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index 80af9dec2..cb9dcad73 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -31,6 +31,11 @@ public Object toObject(int index) throws SFException { return new StructObjectWrapper(object.toString(), object); } + @Override + public byte[] toBytes(int index) throws SFException { + return isNull(index) ? null : toString(index).getBytes(); + } + @Override public String toString(int index) throws SFException { List object = vector.getObject(index); diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index 8212e5830..a60badc81 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -46,6 +46,9 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc .toByteArray(); case Types.VARCHAR: case Types.CHAR: + case Types.STRUCT: + case Types.ARRAY: + case SnowflakeUtil.EXTRA_TYPES_VECTOR: return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java similarity index 94% rename from src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java rename to src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java index 5f7a46ff2..6f701965b 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringArrowJsonCompatibilityIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java @@ -18,14 +18,14 @@ @RunWith(Parameterized.class) @Category(TestCategoryResultSet.class) -public class StructuredTypesGetStringArrowJsonCompatibilityIT +public class StructuredTypesArrowJsonCompatibilityLatestIT extends StructuredTypesGetStringBaseIT { private final String expectedStructureTypeRepresentation; private final String selectSql; private static Map connections = new HashMap<>(); - public StructuredTypesGetStringArrowJsonCompatibilityIT( + public StructuredTypesArrowJsonCompatibilityLatestIT( ResultSetFormatType queryResultFormat, String selectSql, String expectedStructureTypeRepresentation) { @@ -67,6 +67,14 @@ public void testRunAsGetObject() throws SQLException { (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); } + @Test + public void testRunAsGetBytes() throws SQLException { + withFirstRow( + connections.get(queryResultFormat), + selectSql, + (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); + } + @Parameterized.Parameters(name = "format={0},sql={1}") public static Collection data() { Map samples = new LinkedHashMap<>(); diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index ffaca45c8..fb06e7392 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -1,11 +1,16 @@ package net.snowflake.client.jdbc.structuredtypes; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Map; +import java.util.Optional; + import net.snowflake.client.TestUtil; import net.snowflake.client.ThrowingConsumer; import net.snowflake.client.jdbc.BaseJDBCTest; @@ -64,6 +69,12 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) TestUtil.assertEqualsIgnoringWhitespace(expected, resultCasted); } + protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) + throws SQLException { + String result = new String(resultSet.getBytes(1), StandardCharsets.UTF_8); + TestUtil.assertEqualsIgnoringWhitespace(expected, result); + } + protected void withFirstRow( Connection connection, String sqlText, ThrowingConsumer consumer) throws SQLException { From 8f582845bd31121f2d11ff96f3d94b40f6f7bafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Mon, 18 Nov 2024 07:52:21 +0100 Subject: [PATCH 10/29] fix formatting --- .../StructuredTypesArrowJsonCompatibilityLatestIT.java | 3 +-- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java index 6f701965b..cd8cb5ba1 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java @@ -18,8 +18,7 @@ @RunWith(Parameterized.class) @Category(TestCategoryResultSet.class) -public class StructuredTypesArrowJsonCompatibilityLatestIT - extends StructuredTypesGetStringBaseIT { +public class StructuredTypesArrowJsonCompatibilityLatestIT extends StructuredTypesGetStringBaseIT { private final String expectedStructureTypeRepresentation; private final String selectSql; diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index fb06e7392..851b9166e 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -1,6 +1,5 @@ package net.snowflake.client.jdbc.structuredtypes; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; @@ -8,9 +7,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Map; -import java.util.Optional; - import net.snowflake.client.TestUtil; import net.snowflake.client.ThrowingConsumer; import net.snowflake.client.jdbc.BaseJDBCTest; From 9d3d019a4fb60f0aeb3d331a0a69998347dbdc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Mon, 18 Nov 2024 09:04:47 +0100 Subject: [PATCH 11/29] ensure UTF-8 is used for both expected and actual value in string comparison --- src/test/java/net/snowflake/client/TestUtil.java | 6 ++++++ .../structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/snowflake/client/TestUtil.java b/src/test/java/net/snowflake/client/TestUtil.java index ba73dbb01..461a75d67 100644 --- a/src/test/java/net/snowflake/client/TestUtil.java +++ b/src/test/java/net/snowflake/client/TestUtil.java @@ -10,6 +10,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -154,4 +155,9 @@ public static void expectSnowflakeLoggedFeatureNotSupportedException(MethodRaise public static void assertEqualsIgnoringWhitespace(String expected, String actual) { assertEquals(expected.replaceAll("\\s+", ""), actual.replaceAll("\\s+", "")); } + + public static void assertByteStringEqualsIgnoringWhitespace(byte[] expected, byte[] actual) { + assertEqualsIgnoringWhitespace( + new String(expected, StandardCharsets.UTF_8), new String(actual, StandardCharsets.UTF_8)); + } } diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 851b9166e..e2742b11e 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -67,8 +67,8 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { - String result = new String(resultSet.getBytes(1), StandardCharsets.UTF_8); - TestUtil.assertEqualsIgnoringWhitespace(expected, result); + TestUtil.assertByteStringEqualsIgnoringWhitespace( + expected.getBytes(StandardCharsets.UTF_8), resultSet.getBytes(1)); } protected void withFirstRow( From b62a4d93e7bb223cc3ae183383ba4c64a551b33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Mon, 18 Nov 2024 11:57:51 +0100 Subject: [PATCH 12/29] set getBytes charset to UTF-8 --- .../net/snowflake/client/core/arrow/ArrayConverter.java | 3 ++- .../java/net/snowflake/client/core/arrow/MapConverter.java | 3 ++- .../net/snowflake/client/core/arrow/StructConverter.java | 3 ++- .../snowflake/client/core/arrow/VectorTypeConverter.java | 3 ++- src/test/java/net/snowflake/client/TestUtil.java | 6 ------ .../structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ++-- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 96b683151..6f18d06d2 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -1,5 +1,6 @@ package net.snowflake.client.core.arrow; +import java.nio.charset.StandardCharsets; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.arrow.tostringhelpers.ArrowArrayStringRepresentationBuilder; @@ -30,7 +31,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(); + return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index e4e148ac5..0c6ca072e 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -1,5 +1,6 @@ package net.snowflake.client.core.arrow; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -45,7 +46,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(); + return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index ab7d20382..2e5c2e38d 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -1,5 +1,6 @@ package net.snowflake.client.core.arrow; +import java.nio.charset.StandardCharsets; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.SnowflakeJdbcInternalApi; @@ -28,7 +29,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(); + return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index cb9dcad73..af65e8ae6 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -1,5 +1,6 @@ package net.snowflake.client.core.arrow; +import java.nio.charset.StandardCharsets; import java.util.List; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; @@ -33,7 +34,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(); + return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); } @Override diff --git a/src/test/java/net/snowflake/client/TestUtil.java b/src/test/java/net/snowflake/client/TestUtil.java index 461a75d67..ba73dbb01 100644 --- a/src/test/java/net/snowflake/client/TestUtil.java +++ b/src/test/java/net/snowflake/client/TestUtil.java @@ -10,7 +10,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -155,9 +154,4 @@ public static void expectSnowflakeLoggedFeatureNotSupportedException(MethodRaise public static void assertEqualsIgnoringWhitespace(String expected, String actual) { assertEquals(expected.replaceAll("\\s+", ""), actual.replaceAll("\\s+", "")); } - - public static void assertByteStringEqualsIgnoringWhitespace(byte[] expected, byte[] actual) { - assertEqualsIgnoringWhitespace( - new String(expected, StandardCharsets.UTF_8), new String(actual, StandardCharsets.UTF_8)); - } } diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index e2742b11e..392afd84b 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -67,8 +67,8 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { - TestUtil.assertByteStringEqualsIgnoringWhitespace( - expected.getBytes(StandardCharsets.UTF_8), resultSet.getBytes(1)); + TestUtil.assertEqualsIgnoringWhitespace( + expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); } protected void withFirstRow( From 9857661ceeb30752d3baf00081a398e380999d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Mon, 18 Nov 2024 14:42:33 +0100 Subject: [PATCH 13/29] change charset in JSON converter for varchar --- .../java/net/snowflake/client/core/json/BytesConverter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index a60badc81..eb1f80a5a 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -1,6 +1,7 @@ package net.snowflake.client.core.json; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.sql.Types; import net.snowflake.client.core.SFException; import net.snowflake.client.jdbc.ErrorCode; @@ -52,7 +53,7 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) - .getBytes(); + .getBytes(StandardCharsets.UTF_8); case Types.BOOLEAN: return converters.getBooleanConverter().getBoolean(obj, columnType) ? new byte[] {1} From e001bbb915c8803caf622328fa182590d6670ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 09:32:55 +0100 Subject: [PATCH 14/29] use default charset in bytes assert --- .../java/net/snowflake/client/core/arrow/ArrayConverter.java | 2 +- .../java/net/snowflake/client/core/arrow/MapConverter.java | 2 +- .../java/net/snowflake/client/core/arrow/StructConverter.java | 2 +- .../net/snowflake/client/core/arrow/VectorTypeConverter.java | 2 +- .../java/net/snowflake/client/core/json/BytesConverter.java | 2 +- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index 6f18d06d2..e62e68277 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -31,7 +31,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); + return isNull(index) ? null : toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 0c6ca072e..8b77e06b4 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -46,7 +46,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); + return isNull(index) ? null : toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 2e5c2e38d..2e26f4e85 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -29,7 +29,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); + return isNull(index) ? null : toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index af65e8ae6..8c25c311c 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -34,7 +34,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); + return isNull(index) ? null : toString(index).getBytes(); } @Override diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index eb1f80a5a..1f1423dec 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -53,7 +53,7 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) - .getBytes(StandardCharsets.UTF_8); + .getBytes(); case Types.BOOLEAN: return converters.getBooleanConverter().getBoolean(obj, columnType) ? new byte[] {1} diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 392afd84b..ccd6fdb31 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -2,7 +2,7 @@ import static org.junit.Assert.assertTrue; -import java.nio.charset.StandardCharsets; +import java.nio.charset.Charset; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -68,7 +68,7 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { TestUtil.assertEqualsIgnoringWhitespace( - expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); + expected, new String(resultSet.getBytes(1), Charset.defaultCharset())); } protected void withFirstRow( From 04a476d1c5b9381d6c70e00eebfe9d4ee66b6516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 09:46:40 +0100 Subject: [PATCH 15/29] fix formatting --- .../java/net/snowflake/client/core/arrow/ArrayConverter.java | 1 - src/main/java/net/snowflake/client/core/arrow/MapConverter.java | 1 - .../java/net/snowflake/client/core/arrow/StructConverter.java | 1 - .../net/snowflake/client/core/arrow/VectorTypeConverter.java | 1 - src/main/java/net/snowflake/client/core/json/BytesConverter.java | 1 - 5 files changed, 5 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java index e62e68277..96b683151 100644 --- a/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/ArrayConverter.java @@ -1,6 +1,5 @@ package net.snowflake.client.core.arrow; -import java.nio.charset.StandardCharsets; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.arrow.tostringhelpers.ArrowArrayStringRepresentationBuilder; diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index 8b77e06b4..e4e148ac5 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -1,6 +1,5 @@ package net.snowflake.client.core.arrow; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java index 2e26f4e85..ab7d20382 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructConverter.java @@ -1,6 +1,5 @@ package net.snowflake.client.core.arrow; -import java.nio.charset.StandardCharsets; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; import net.snowflake.client.core.SnowflakeJdbcInternalApi; diff --git a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java index 8c25c311c..cb9dcad73 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VectorTypeConverter.java @@ -1,6 +1,5 @@ package net.snowflake.client.core.arrow; -import java.nio.charset.StandardCharsets; import java.util.List; import net.snowflake.client.core.DataConversionContext; import net.snowflake.client.core.SFException; diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index 1f1423dec..a60badc81 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -1,7 +1,6 @@ package net.snowflake.client.core.json; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.sql.Types; import net.snowflake.client.core.SFException; import net.snowflake.client.jdbc.ErrorCode; From 55e7135a4a9907bdbee827997b10aaac76344102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 14:22:17 +0100 Subject: [PATCH 16/29] ensure UTF-8 in getBytes assert --- .../structuredtypes/StructuredTypesGetStringBaseIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index ccd6fdb31..9d7f972a6 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -2,7 +2,7 @@ import static org.junit.Assert.assertTrue; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -67,8 +67,10 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { + byte[] expectedBytes = expected.getBytes(); TestUtil.assertEqualsIgnoringWhitespace( - expected, new String(resultSet.getBytes(1), Charset.defaultCharset())); + new String(expectedBytes, StandardCharsets.UTF_8), + new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); } protected void withFirstRow( From f911113b217f3010a95bb784eae5d87ba7ac749a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 14:55:05 +0100 Subject: [PATCH 17/29] remove charset from assert --- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 9d7f972a6..fc5f5df49 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -2,7 +2,6 @@ import static org.junit.Assert.assertTrue; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -69,8 +68,7 @@ protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { byte[] expectedBytes = expected.getBytes(); TestUtil.assertEqualsIgnoringWhitespace( - new String(expectedBytes, StandardCharsets.UTF_8), - new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); + new String(expectedBytes), new String(resultSet.getBytes(1))); } protected void withFirstRow( From 4681524d84976f3f315b88414d3f059d7ca3e9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 15:48:40 +0100 Subject: [PATCH 18/29] remove UTF-8 charset specification from VarCharConverter --- .../java/net/snowflake/client/core/arrow/VarCharConverter.java | 2 +- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java b/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java index b53595d42..934c8571f 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java @@ -35,7 +35,7 @@ public VarCharConverter(ValueVector valueVector, int columnIndex, DataConversion @Override public String toString(int index) { byte[] bytes = toBytes(index); - return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8); + return bytes == null ? null : new String(bytes); } @Override diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index fc5f5df49..25f3fb1c4 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -66,9 +66,8 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { - byte[] expectedBytes = expected.getBytes(); TestUtil.assertEqualsIgnoringWhitespace( - new String(expectedBytes), new String(resultSet.getBytes(1))); + expected, new String(resultSet.getBytes(1))); } protected void withFirstRow( From 2f0032c0bac0e051255a16db7f91f75b190695cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 16:00:34 +0100 Subject: [PATCH 19/29] add charset to both VarcharConverter and BytesConverter for Varchar type --- .../java/net/snowflake/client/core/arrow/VarCharConverter.java | 2 +- .../java/net/snowflake/client/core/json/BytesConverter.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java b/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java index 934c8571f..b53595d42 100644 --- a/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/VarCharConverter.java @@ -35,7 +35,7 @@ public VarCharConverter(ValueVector valueVector, int columnIndex, DataConversion @Override public String toString(int index) { byte[] bytes = toBytes(index); - return bytes == null ? null : new String(bytes); + return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8); } @Override diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index a60badc81..eb1f80a5a 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -1,6 +1,7 @@ package net.snowflake.client.core.json; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.sql.Types; import net.snowflake.client.core.SFException; import net.snowflake.client.jdbc.ErrorCode; @@ -52,7 +53,7 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) - .getBytes(); + .getBytes(StandardCharsets.UTF_8); case Types.BOOLEAN: return converters.getBooleanConverter().getBoolean(obj, columnType) ? new byte[] {1} From 67e001bf2f0f4cea14033e9996a2e7cdad1bad2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 19 Nov 2024 17:45:39 +0100 Subject: [PATCH 20/29] assert charset is utf-8 --- .../java/net/snowflake/client/core/arrow/MapConverter.java | 3 ++- .../java/net/snowflake/client/core/json/BytesConverter.java | 3 ++- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java index e4e148ac5..0c6ca072e 100644 --- a/src/main/java/net/snowflake/client/core/arrow/MapConverter.java +++ b/src/main/java/net/snowflake/client/core/arrow/MapConverter.java @@ -1,5 +1,6 @@ package net.snowflake.client.core.arrow; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -45,7 +46,7 @@ public Object toObject(int index) throws SFException { @Override public byte[] toBytes(int index) throws SFException { - return isNull(index) ? null : toString(index).getBytes(); + return isNull(index) ? null : toString(index).getBytes(StandardCharsets.UTF_8); } @Override diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index eb1f80a5a..1600da0b9 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -1,6 +1,7 @@ package net.snowflake.client.core.json; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.sql.Types; import net.snowflake.client.core.SFException; @@ -53,7 +54,7 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) - .getBytes(StandardCharsets.UTF_8); + .getBytes(); case Types.BOOLEAN: return converters.getBooleanConverter().getBoolean(obj, columnType) ? new byte[] {1} diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 25f3fb1c4..c8d18a051 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -1,7 +1,10 @@ package net.snowflake.client.jdbc.structuredtypes; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -66,6 +69,7 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { + assertEquals(StandardCharsets.UTF_8, Charset.defaultCharset()); TestUtil.assertEqualsIgnoringWhitespace( expected, new String(resultSet.getBytes(1))); } From f8512456f2b419aa827d314a84cfb8ab3c6b2a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Wed, 20 Nov 2024 10:25:54 +0100 Subject: [PATCH 21/29] add charset specification --- .../java/net/snowflake/client/core/json/BytesConverter.java | 3 +-- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/json/BytesConverter.java b/src/main/java/net/snowflake/client/core/json/BytesConverter.java index 1600da0b9..eb1f80a5a 100644 --- a/src/main/java/net/snowflake/client/core/json/BytesConverter.java +++ b/src/main/java/net/snowflake/client/core/json/BytesConverter.java @@ -1,7 +1,6 @@ package net.snowflake.client.core.json; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.sql.Types; import net.snowflake.client.core.SFException; @@ -54,7 +53,7 @@ public byte[] getBytes(Object obj, int columnType, int columnSubType, Integer sc return converters .getStringConverter() .getString(obj, columnType, columnSubType, scale) - .getBytes(); + .getBytes(StandardCharsets.UTF_8); case Types.BOOLEAN: return converters.getBooleanConverter().getBoolean(obj, columnType) ? new byte[] {1} diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index c8d18a051..392afd84b 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -1,9 +1,7 @@ package net.snowflake.client.jdbc.structuredtypes; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; @@ -69,9 +67,8 @@ protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) throws SQLException { - assertEquals(StandardCharsets.UTF_8, Charset.defaultCharset()); TestUtil.assertEqualsIgnoringWhitespace( - expected, new String(resultSet.getBytes(1))); + expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); } protected void withFirstRow( From 6a0631152b04c84f0128e839637df2ffcc5f933c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 22 Nov 2024 14:31:45 +0100 Subject: [PATCH 22/29] add license to struct object wrapper, reuse constructor for SfSqlArray, simplify getObject conditions --- .../java/net/snowflake/client/core/SFArrowResultSet.java | 5 ++--- src/main/java/net/snowflake/client/core/SfSqlArray.java | 3 +-- .../net/snowflake/client/core/arrow/StructObjectWrapper.java | 3 +++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 63267302d..38dd77ab5 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -577,10 +577,9 @@ public Object getObject(int columnIndex) throws SFException { Object obj = converter.toObject(index); boolean isStructuredType = resultSetMetaData.isStructuredTypeColumn(columnIndex); if (type == Types.STRUCT && isStructuredType && converter instanceof VarCharConverter) { - if (obj == null) { - return null; + if (obj != null) { + return new StructObjectWrapper((String) obj, createJsonSqlInput(columnIndex, obj)); } - return new StructObjectWrapper((String) obj, createJsonSqlInput(columnIndex, obj)); } return obj; } diff --git a/src/main/java/net/snowflake/client/core/SfSqlArray.java b/src/main/java/net/snowflake/client/core/SfSqlArray.java index 5d3675cf0..c98db4473 100644 --- a/src/main/java/net/snowflake/client/core/SfSqlArray.java +++ b/src/main/java/net/snowflake/client/core/SfSqlArray.java @@ -27,8 +27,7 @@ public SfSqlArray(String text, int baseType, Object elements) { } public SfSqlArray(int baseType, Object elements) { - this.baseType = baseType; - this.elements = elements; + this(null, baseType, elements); } @Override diff --git a/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java b/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java index b69b4645e..8219c110a 100644 --- a/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java +++ b/src/main/java/net/snowflake/client/core/arrow/StructObjectWrapper.java @@ -1,3 +1,6 @@ +/* + * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. + */ package net.snowflake.client.core.arrow; import net.snowflake.client.core.SnowflakeJdbcInternalApi; From 496f99ca0df8dc43bd356918d9cd9fc25298b7c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 22 Nov 2024 15:17:22 +0100 Subject: [PATCH 23/29] rename obj to arrayString in getJsonArray --- .../client/core/SFBaseResultSet.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFBaseResultSet.java b/src/main/java/net/snowflake/client/core/SFBaseResultSet.java index e13ceb2fb..c0b6256ad 100644 --- a/src/main/java/net/snowflake/client/core/SFBaseResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFBaseResultSet.java @@ -273,7 +273,7 @@ protected SQLInput createJsonSqlInputForColumn( } @SnowflakeJdbcInternalApi - protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFException { + protected SfSqlArray getJsonArray(String arrayString, int columnIndex) throws SFException { try { List fieldMetadataList = resultSetMetaData.getColumnFields(columnIndex); if (fieldMetadataList.size() != 1) { @@ -288,38 +288,38 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio int columnType = ColumnTypeHelper.getColumnType(columnSubType, session); int scale = fieldMetadata.getScale(); - ArrayNode arrayNode = (ArrayNode) OBJECT_MAPPER.readTree(obj); + ArrayNode arrayNode = (ArrayNode) OBJECT_MAPPER.readTree(arrayString); Iterator nodeElements = arrayNode.elements(); switch (columnType) { case Types.INTEGER: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().integerConverter(columnType)) .toArray(Integer[]::new)); case Types.SMALLINT: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().smallIntConverter(columnType)) .toArray(Short[]::new)); case Types.TINYINT: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().tinyIntConverter(columnType)) .toArray(Byte[]::new)); case Types.BIGINT: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().bigIntConverter(columnType)) .toArray(Long[]::new)); case Types.DECIMAL: case Types.NUMERIC: return new SfSqlArray( - obj, + arrayString, columnSubType, convertToFixedArray( getStream(nodeElements, getConverters().bigDecimalConverter(columnType)))); @@ -327,7 +327,7 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio case Types.VARCHAR: case Types.LONGNVARCHAR: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream( nodeElements, @@ -335,38 +335,38 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio .toArray(String[]::new)); case Types.BINARY: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().bytesConverter(columnType, scale)) .toArray(Byte[][]::new)); case Types.FLOAT: case Types.REAL: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().floatConverter(columnType)) .toArray(Float[]::new)); case Types.DOUBLE: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().doubleConverter(columnType)) .toArray(Double[]::new)); case Types.DATE: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().dateStringConverter(session)) .toArray(Date[]::new)); case Types.TIME: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().timeFromStringConverter(session)) .toArray(Time[]::new)); case Types.TIMESTAMP: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream( nodeElements, @@ -376,19 +376,19 @@ protected SfSqlArray getJsonArray(String obj, int columnIndex) throws SFExceptio .toArray(Timestamp[]::new)); case Types.BOOLEAN: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().booleanConverter(columnType)) .toArray(Boolean[]::new)); case Types.STRUCT: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().structConverter(OBJECT_MAPPER)) .toArray(Map[]::new)); case Types.ARRAY: return new SfSqlArray( - obj, + arrayString, columnSubType, getStream(nodeElements, getConverters().arrayConverter(OBJECT_MAPPER)) .toArray(Map[][]::new)); From 2b9636d0befc25510b9ac3743782925a74a36731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 26 Nov 2024 10:25:38 +0100 Subject: [PATCH 24/29] merge getString getBytes and getObject test into one method to reuse queried resultSet --- ...edTypesArrowJsonCompatibilityLatestIT.java | 22 ++++--------------- .../StructuredTypesGetStringBaseIT.java | 17 ++++++-------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java index cd8cb5ba1..c4c870e08 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java @@ -1,6 +1,7 @@ package net.snowflake.client.jdbc.structuredtypes; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -23,6 +24,7 @@ public class StructuredTypesArrowJsonCompatibilityLatestIT extends StructuredTyp private final String expectedStructureTypeRepresentation; private final String selectSql; private static Map connections = new HashMap<>(); + private static Map> resultSets = new HashMap<>(); public StructuredTypesArrowJsonCompatibilityLatestIT( ResultSetFormatType queryResultFormat, @@ -51,27 +53,11 @@ public static void closeConnections() throws SQLException { } @Test - public void testRunAsGetString() throws SQLException { + public void testArrowJsonCompatibility() throws SQLException { withFirstRow( connections.get(queryResultFormat), selectSql, - (resultSet) -> assertGetStringIsCompatible(resultSet, expectedStructureTypeRepresentation)); - } - - @Test - public void testRunAsGetObject() throws SQLException { - withFirstRow( - connections.get(queryResultFormat), - selectSql, - (resultSet) -> assertGetObjectIsCompatible(resultSet, expectedStructureTypeRepresentation)); - } - - @Test - public void testRunAsGetBytes() throws SQLException { - withFirstRow( - connections.get(queryResultFormat), - selectSql, - (resultSet) -> assertGetBytesIsCompatible(resultSet, expectedStructureTypeRepresentation)); + (resultSet) -> assertResultSetIsCompatible(resultSet, expectedStructureTypeRepresentation)); } @Parameterized.Parameters(name = "format={0},sql={1}") diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 392afd84b..5b56d2d96 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -51,24 +51,21 @@ protected static Connection initConnection(ResultSetFormatType queryResultFormat return conn; } - protected void assertGetStringIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + protected void assertResultSetIsCompatible(ResultSet resultSet, String expected) + throws SQLException { + // Test getString String result = resultSet.getString(1); TestUtil.assertEqualsIgnoringWhitespace(expected, result); - } - protected void assertGetObjectIsCompatible(ResultSet resultSet, String expected) - throws SQLException { - String result = resultSet.getObject(1, String.class); + // Test getObject + result = resultSet.getObject(1, String.class); String resultCasted = (String) resultSet.getObject(1); TestUtil.assertEqualsIgnoringWhitespace(expected, result); TestUtil.assertEqualsIgnoringWhitespace(expected, resultCasted); - } - protected void assertGetBytesIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + // Test getBytes TestUtil.assertEqualsIgnoringWhitespace( - expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); + expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); } protected void withFirstRow( From b2e1136b71ec9a1db098d8ead8ea058cc1cd875d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 26 Nov 2024 10:36:21 +0100 Subject: [PATCH 25/29] fix formatting --- .../jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java index 5b56d2d96..05de55f99 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesGetStringBaseIT.java @@ -52,7 +52,7 @@ protected static Connection initConnection(ResultSetFormatType queryResultFormat } protected void assertResultSetIsCompatible(ResultSet resultSet, String expected) - throws SQLException { + throws SQLException { // Test getString String result = resultSet.getString(1); TestUtil.assertEqualsIgnoringWhitespace(expected, result); @@ -65,7 +65,7 @@ protected void assertResultSetIsCompatible(ResultSet resultSet, String expected) // Test getBytes TestUtil.assertEqualsIgnoringWhitespace( - expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); + expected, new String(resultSet.getBytes(1), StandardCharsets.UTF_8)); } protected void withFirstRow( From 5dff05bef801a8058dbc648e2b5e2910fed5d0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 26 Nov 2024 14:53:31 +0100 Subject: [PATCH 26/29] split getJsonString in SfSqlArray into getJsonString for writes and get(predefined)Text for reads, extract isVarcharConvertedStruct --- .../snowflake/client/core/SFArrowResultSet.java | 7 ++++++- .../java/net/snowflake/client/core/SfSqlArray.java | 14 ++++++++++++-- .../client/jdbc/SnowflakeResultSetV1.java | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java index 38dd77ab5..69195e8a4 100644 --- a/src/main/java/net/snowflake/client/core/SFArrowResultSet.java +++ b/src/main/java/net/snowflake/client/core/SFArrowResultSet.java @@ -576,7 +576,7 @@ public Object getObject(int columnIndex) throws SFException { converter.setSessionTimeZone(sessionTimeZone); Object obj = converter.toObject(index); boolean isStructuredType = resultSetMetaData.isStructuredTypeColumn(columnIndex); - if (type == Types.STRUCT && isStructuredType && converter instanceof VarCharConverter) { + if (isVarcharConvertedStruct(type, isStructuredType, converter)) { if (obj != null) { return new StructObjectWrapper((String) obj, createJsonSqlInput(columnIndex, obj)); } @@ -584,6 +584,11 @@ public Object getObject(int columnIndex) throws SFException { return obj; } + private boolean isVarcharConvertedStruct( + int type, boolean isStructuredType, ArrowVectorConverter converter) { + return type == Types.STRUCT && isStructuredType && converter instanceof VarCharConverter; + } + private Object createJsonSqlInput(int columnIndex, Object obj) throws SFException { try { if (obj == null) { diff --git a/src/main/java/net/snowflake/client/core/SfSqlArray.java b/src/main/java/net/snowflake/client/core/SfSqlArray.java index c98db4473..4966d7ab8 100644 --- a/src/main/java/net/snowflake/client/core/SfSqlArray.java +++ b/src/main/java/net/snowflake/client/core/SfSqlArray.java @@ -1,6 +1,7 @@ package net.snowflake.client.core; import static net.snowflake.client.core.FieldSchemaCreator.buildBindingSchemaForType; +import static net.snowflake.client.core.FieldSchemaCreator.logger; import com.fasterxml.jackson.core.JsonProcessingException; import java.sql.Array; @@ -19,6 +20,7 @@ public class SfSqlArray implements Array { private String text; private int baseType; private Object elements; + private String jsonStringFromElements; public SfSqlArray(String text, int baseType, Object elements) { this.text = text; @@ -87,13 +89,21 @@ public ResultSet getResultSet(long index, int count, Map> map) @Override public void free() throws SQLException {} - public String getJsonString() throws SQLException { + public String getText() { if (text == null) { - text = buildJsonStringFromElements(elements); + logger.warn("Text field wasn't initialized. Should never happen."); } return text; } + public String getJsonString() throws SQLException { + if (jsonStringFromElements == null) { + jsonStringFromElements = buildJsonStringFromElements(elements); + } + + return jsonStringFromElements; + } + private static String buildJsonStringFromElements(Object elements) throws SQLException { try { return SnowflakeUtil.mapJson(elements); diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java index dfe8cbad6..e9db5ec71 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeResultSetV1.java @@ -279,7 +279,7 @@ public Object getObject(int columnIndex) throws SQLException { } else if (object instanceof StructObjectWrapper) { return ((StructObjectWrapper) object).getJsonString(); } else if (object instanceof SfSqlArray) { - return ((SfSqlArray) object).getJsonString(); + return ((SfSqlArray) object).getText(); } else if (object instanceof ArrowSqlInput) { throw new SQLException( "Arrow native struct couldn't be converted to String. To map to SqlData the method getObject(int columnIndex, Class type) should be used"); From fedace417a442595cdb69a16fa97fea1b56f0e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Wed, 27 Nov 2024 17:40:04 +0100 Subject: [PATCH 27/29] remove incorrect dependency, fix filename after resolving merge conflicts --- .../ResultSetStructuredTypesLatestIT.java | 59 +++++++++---------- ...edTypesArrowJsonCompatibilityLatestIT.java | 5 +- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java index 55c2bb89f..ed2706b93 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java @@ -224,40 +224,39 @@ public void testReturnStructAsStringIfTypeWasNotIndicated(ResultSetFormatType fo + " \"intValue\": 2\n" + " }\n" + "}"; - String expectedJsonFromArrow = - "{\"string\": \"a\",\"b\": 1,\"s\": 2,\"i\": 3,\"l\": 4,\"f\": 1.1,\"d\": 2.2,\"bd\": 3.3," - + "\"bool\": true,\"timestamp_ltz\": \"2021-12-22 09:43:44.000 +0100\",\"timestamp_ntz\": \"2021-12-23 09:44:44.000\"," - + "\"timestamp_tz\": \"2021-12-24 09:45:45.000 +0800\",\"date\": \"2023-12-24\",\"time\": \"12:34:56\",\"binary\": \"616263\"," - + "\"simpleClass\": {\"string\": \"b\",\"intValue\": 2}}"; - if (format == ResultSetFormatType.NATIVE_ARROW) { - Assert.assertEquals(expectedJsonFromArrow, object); - } else { - Assert.assertEquals(expectedJson, object); - } + String expectedJsonFromArrow = + "{\"string\": \"a\",\"b\": 1,\"s\": 2,\"i\": 3,\"l\": 4,\"f\": 1.1,\"d\": 2.2,\"bd\": 3.3," + + "\"bool\": true,\"timestamp_ltz\": \"2021-12-22 09:43:44.000 +0100\",\"timestamp_ntz\": \"2021-12-23 09:44:44.000\"," + + "\"timestamp_tz\": \"2021-12-24 09:45:45.000 +0800\",\"date\": \"2023-12-24\",\"time\": \"12:34:56\",\"binary\": \"616263\"," + + "\"simpleClass\": {\"string\": \"b\",\"intValue\": 2}}"; + if (format == ResultSetFormatType.NATIVE_ARROW) { + Assert.assertEquals(expectedJsonFromArrow, object); + } else { + Assert.assertEquals(expectedJson, object); + } } } } - - @ParameterizedTest - @ArgumentsSource(ResultFormatProvider.class) - @DontRunOnGithubActions - public void testThrowingGettingObjectIfTypeWasNotIndicatedAndFormatNativeArrow( - ResultSetFormatType format) throws SQLException { - Assumptions.assumeTrue(format == ResultSetFormatType.NATIVE_ARROW); - withFirstRow( - "select {'string':'a'}::OBJECT(string VARCHAR)", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1)); - }, - format); - withFirstRow( - "select {'x':{'string':'one'},'y':{'string':'two'},'z':{'string':'three'}}::MAP(VARCHAR, OBJECT(string VARCHAR));", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1, Map.class)); - }, - format); - } + @ParameterizedTest + @ArgumentsSource(ResultFormatProvider.class) + @DontRunOnGithubActions + public void testThrowingGettingObjectIfTypeWasNotIndicatedAndFormatNativeArrow( + ResultSetFormatType format) throws SQLException { + Assumptions.assumeTrue(format == ResultSetFormatType.NATIVE_ARROW); + withFirstRow( + "select {'string':'a'}::OBJECT(string VARCHAR)", + (resultSet) -> { + assertThrows(SQLException.class, () -> resultSet.getObject(1)); + }, + format); + withFirstRow( + "select {'x':{'string':'one'},'y':{'string':'two'},'z':{'string':'three'}}::MAP(VARCHAR, OBJECT(string VARCHAR));", + (resultSet) -> { + assertThrows(SQLException.class, () -> resultSet.getObject(1, Map.class)); + }, + format); + } @ParameterizedTest @ArgumentsSource(ResultFormatProvider.class) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java index 1593c13ee..77bfca52d 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/StructuredTypesArrowJsonCompatibilityLatestIT.java @@ -6,8 +6,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; - -import jdk.jpackage.internal.Arguments; import net.snowflake.client.annotations.DontRunOnGithubActions; import net.snowflake.client.category.TestTags; import net.snowflake.client.jdbc.ResultSetFormatType; @@ -23,8 +21,7 @@ import org.junit.jupiter.params.provider.ArgumentsSource; @Tag(TestTags.RESULT_SET) -public class StructuredTypesGetStringArrowJsonCompatibilityIT - extends StructuredTypesGetStringBaseIT { +public class StructuredTypesArrowJsonCompatibilityLatestIT extends StructuredTypesGetStringBaseIT { private static Map connections = new HashMap<>(); @BeforeAll From 1d864ee0f7118d35f9a2b881f2ba8f48b14d68fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 29 Nov 2024 14:49:23 +0100 Subject: [PATCH 28/29] remove testThrowingGettingObjectIfTypeWasNotIndicatedAndFormatNativeArrow test that was accidentally added while resolving conflicts --- .../ResultSetStructuredTypesLatestIT.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java index ed2706b93..2ca95bc48 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java @@ -238,26 +238,6 @@ public void testReturnStructAsStringIfTypeWasNotIndicated(ResultSetFormatType fo } } - @ParameterizedTest - @ArgumentsSource(ResultFormatProvider.class) - @DontRunOnGithubActions - public void testThrowingGettingObjectIfTypeWasNotIndicatedAndFormatNativeArrow( - ResultSetFormatType format) throws SQLException { - Assumptions.assumeTrue(format == ResultSetFormatType.NATIVE_ARROW); - withFirstRow( - "select {'string':'a'}::OBJECT(string VARCHAR)", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1)); - }, - format); - withFirstRow( - "select {'x':{'string':'one'},'y':{'string':'two'},'z':{'string':'three'}}::MAP(VARCHAR, OBJECT(string VARCHAR));", - (resultSet) -> { - assertThrows(SQLException.class, () -> resultSet.getObject(1, Map.class)); - }, - format); - } - @ParameterizedTest @ArgumentsSource(ResultFormatProvider.class) @DontRunOnGithubActions From aae2c84e4072444ebf97471e3c60e6d395fc2d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 29 Nov 2024 15:15:33 +0100 Subject: [PATCH 29/29] fix formatting --- .../jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java index 2ca95bc48..f9b3368d8 100644 --- a/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/structuredtypes/ResultSetStructuredTypesLatestIT.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal;