#119: Fix joining HASHTYPE columns

.project-keeper.yml

@@ -9,4 +9,4 @@ sources:
 build:
   runnerOs: ubuntu-20.04
   exasolDbVersions:
-    - "7.1.24"
+    - "7.1.25"

doc/changes/changes_7.2.0.md

@@ -0,0 +1,42 @@
+# Exasol Virtual Schema 7.2.0, released 2024-02-22
+
+Code name: Add parameter `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA`
+
+## Summary
+
+Using `IMPORT FROM EXA` might lead to some unexpected datatype mappings. Unlike for a JDBC connection there's no explicit data mapping being generated when using `IMPORT FROM EXA`. The Exasol specific types `GEOMETRY`, `INTERVAL YEAR TO MONTH`, `INTERVAL DAY TO SECOND` and `HASHTYPE` are mapped to `VARCHAR`. This release adds parameter `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA`. When setting this to `true`, the data types are mapped as expected.
+
+Setting `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA` to `true` also fixes a bug when joining a table in a virtual schema with a normal table using a `HASHTYPE` column. This failed before in Exasol 7.1 with error message `Feature not supported: Incomparable Types: VARCHAR(32) UTF8 and HASHTYPE(16 BYTE)!`.
+
+See the [user guide](../dialects/exasol.md#auto-generated-datatype-mapping-using-exa-import) for details.
+
+This release also fixes vulnerabilities CVE-2024-25710 and CVE-2024-26308 in transitive test dependency `org.apache.commons:commons-compress`.
+
+## Security
+
+* #120: Fixed CVE-2024-25710 in `org.apache.commons:commons-compress`
+* #121: Fixed CVE-2024-26308 in `org.apache.commons:commons-compress`
+
+## Bugfixes
+
+* #119: Fixed data types for `IMPORT FROM EXA`
+
+## Dependency Updates
+
+### Test Dependency Updates
+
+* Updated `com.exasol:exasol-testcontainers:6.6.3` to `7.0.1`
+* Updated `com.exasol:hamcrest-resultset-matcher:1.6.3` to `1.6.4`
+* Updated `com.exasol:test-db-builder-java:3.5.2` to `3.5.3`
+* Updated `org.junit.jupiter:junit-jupiter:5.10.1` to `5.10.2`
+* Updated `org.mockito:mockito-junit-jupiter:5.7.0` to `5.10.0`
+* Updated `org.slf4j:slf4j-jdk14:2.0.9` to `2.0.12`
+* Updated `org.testcontainers:junit-jupiter:1.19.2` to `1.19.6`
+
+### Plugin Dependency Updates
+
+* Updated `com.exasol:project-keeper-maven-plugin:2.9.16` to `3.0.1`
+* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.2.2` to `3.2.3`
+* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.2.2` to `3.2.3`
+* Added `org.apache.maven.plugins:maven-toolchains-plugin:3.1.0`
+* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.1` to `2.16.2`

doc/dialects/exasol.md

@@ -17,7 +17,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_EXASOL AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtual-schema-dist-11.0.2-exasol-7.1.7.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtual-schema-dist-11.0.2-exasol-7.2.0.jar;
 /
 ```
 
@@ -84,6 +84,34 @@ USING SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_EXASOL WITH
     EXA_CONNECTION  = 'EXA_CONNECTION';
 ```
 
+#### Auto-generated Datatype Mapping Using EXA Import
+
+Using `IMPORT FROM EXA` might lead to some unexpected datatype mappings. Unlike for a JDBC connection there's no explicit data mapping being generated when using `IMPORT FROM EXA`. As a solution the Exasol Virtual Schema in version 7.2.0 and later supports parameter `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA`:
+
+```sql
+CREATE VIRTUAL SCHEMA VIRTUAL_EXASOL
+USING SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_EXASOL WITH
+    CONNECTION_NAME                        = 'JDBC_CONNECTION'
+    SCHEMA_NAME                            = '<schema name>'
+    IMPORT_FROM_EXA                        = 'true'
+    EXA_CONNECTION                         = 'EXA_CONNECTION'
+    GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA = 'true';
+```
+
+This will add explicit datatype mapping to the generated command when using `IMPORT FROM EXA`.
+
+Example for the generated pushdown query with `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA = 'false'`:
+
+```sql
+IMPORT FROM EXA AT "EXA_CONNECTION" STATEMENT '...'
+```
+
+Pushdown query with `GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA = 'true'`:
+
+```sql
+IMPORT INTO (c1 DECIMAL(36,1), c2 .... ) FROM EXA AT "EXA_CONNECTION" STATEMENT '...'
+```
+
 ### Using `IMPORT FROM JDBC`
 
 You can alternatively use a regular JDBC connection for the `IMPORT`. Note that this option is slower because it lacks the parallelization the `IMPORT FROM EXA` variant.

pom.xml

@@ -2,7 +2,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>exasol-virtual-schema</artifactId>
-    <version>7.1.7</version>
+    <version>7.2.0</version>
     <name>Exasol Virtual Schema</name>
     <description>This projects contains the Exasol dialect for Exasol's Virtual Schema</description>
     <url>https://github.com/exasol/exasol-virtual-schema/</url>
@@ -43,38 +43,38 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <version>5.10.1</version>
+            <version>5.10.2</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-junit-jupiter</artifactId>
-            <version>5.7.0</version>
+            <version>5.10.0</version>
             <scope>test</scope>
         </dependency>
         <!--Integration test dependencies -->
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>exasol-testcontainers</artifactId>
-            <version>6.6.3</version>
+            <version>7.0.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <version>1.19.2</version>
+            <version>1.19.6</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>hamcrest-resultset-matcher</artifactId>
-            <version>1.6.3</version>
+            <version>1.6.4</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-jdk14</artifactId>
-            <version>2.0.9</version>
+            <version>2.0.12</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -175,7 +175,7 @@
     <parent>
         <artifactId>exasol-virtual-schema-generated-parent</artifactId>
         <groupId>com.exasol</groupId>
-        <version>7.1.7</version>
+        <version>7.2.0</version>
         <relativePath>pk_generated_parent.pom</relativePath>
     </parent>
 </project>

src/main/java/com/exasol/adapter/dialects/exasol/ExasolConnectionDefinitionBuilder.java

@@ -44,4 +44,4 @@ private String buildImportFromExaConnectionDefinition(final AdapterProperties pr
     private String getExasolConnectionName(final AdapterProperties properties) {
         return properties.get(EXASOL_CONNECTION_PROPERTY);
     }
-}
+}

src/main/java/com/exasol/adapter/dialects/exasol/ExasolFromExaQueryRewriter.java

@@ -1,57 +1,27 @@
 package com.exasol.adapter.dialects.exasol;
 
-import java.sql.SQLException;
-import java.util.List;
-import java.util.logging.Logger;
-
 import com.exasol.adapter.dialects.SqlDialect;
 import com.exasol.adapter.dialects.rewriting.AbstractQueryRewriter;
-import com.exasol.adapter.dialects.rewriting.SqlGenerationHelper;
-import com.exasol.adapter.jdbc.*;
-import com.exasol.adapter.metadata.DataType;
+import com.exasol.adapter.jdbc.RemoteMetadataReader;
 
 /**
- * Exasol-specific query rewriter for {@code IMPORT FROM EXA}. It is similar to {@link ExasolJdbcQueryRewriter} but uses
- * {@code IMPORT INTO (...) FROM EXA}.
+ * Exasol-specific query rewriter for {@code IMPORT FROM EXA} that does not add data types to the pushdown query. Data
+ * types like {@code HASHTYPE} will be reported as {@code VARCHAR}.
  */
-class ExasolFromExaQueryRewriter extends AbstractQueryRewriter {
-
-    private static final Logger LOGGER = Logger.getLogger(ExasolFromExaQueryRewriter.class.getName());
-    private final ConnectionFactory connectionFactory;
-
-    ExasolFromExaQueryRewriter(final SqlDialect dialect, final RemoteMetadataReader remoteMetadataReader,
-            final ConnectionFactory connectionFactory) {
+public class ExasolFromExaQueryRewriter extends AbstractQueryRewriter {
+
+    /**
+     * Create a new instance of the {@link ExasolFromExaQueryRewriter}.
+     *
+     * @param dialect              dialect
+     * @param remoteMetadataReader remote metadata reader
+     */
+    public ExasolFromExaQueryRewriter(final SqlDialect dialect, final RemoteMetadataReader remoteMetadataReader) {
         super(dialect, remoteMetadataReader, new ExasolConnectionDefinitionBuilder());
-        this.connectionFactory = connectionFactory;
-    }
-
-    @Override
-    protected String generateImportStatement(final String connectionDefinition,
-            final List<DataType> selectListDataTypes, final String pushdownQuery) throws SQLException {
-        return generateImportStatement(SqlGenerationHelper.createColumnsDescriptionFromDataTypes(selectListDataTypes),
-                connectionDefinition, pushdownQuery);
     }
 
     @Override
-    protected String generateImportStatement(final String connectionDefinition, final String pushdownQuery)
-            throws SQLException {
-        return generateImportStatement(createColumnsDescriptionFromQuery(pushdownQuery), connectionDefinition,
-                pushdownQuery);
-    }
-
-    private String generateImportStatement(final String columnsDescription, final String connectionDefinition,
-            final String pushdownQuery) {
-        return "IMPORT INTO (" + columnsDescription + ") FROM EXA " //
-                + connectionDefinition + " STATEMENT '" //
-                + pushdownQuery.replace("'", "''") + "'";
-    }
-
-    private String createColumnsDescriptionFromQuery(final String query) throws SQLException {
-        final ColumnMetadataReader columnMetadataReader = this.remoteMetadataReader.getColumnMetadataReader();
-        final ResultSetMetadataReader resultSetMetadataReader = new ResultSetMetadataReader(
-                this.connectionFactory.getConnection(), columnMetadataReader);
-        final String columnsDescription = resultSetMetadataReader.describeColumns(query);
-        LOGGER.finer(() -> "Import columns: " + columnsDescription);
-        return columnsDescription;
+    protected String generateImportStatement(final String connectionDefinition, final String pushdownQuery) {
+        return "IMPORT FROM EXA " + connectionDefinition + " STATEMENT '" + pushdownQuery.replace("'", "''") + "'";
     }
 }

src/main/java/com/exasol/adapter/dialects/exasol/ExasolFromExaWithDataTypeQueryRewriter.java

@@ -0,0 +1,59 @@
+package com.exasol.adapter.dialects.exasol;
+
+import java.sql.SQLException;
+import java.util.List;
+import java.util.logging.Logger;
+
+import com.exasol.adapter.dialects.SqlDialect;
+import com.exasol.adapter.dialects.rewriting.AbstractQueryRewriter;
+import com.exasol.adapter.dialects.rewriting.SqlGenerationHelper;
+import com.exasol.adapter.jdbc.*;
+import com.exasol.adapter.metadata.DataType;
+
+/**
+ * Exasol-specific query rewriter for {@code IMPORT FROM EXA} that adds data types to the pushdown query. Data types
+ * like {@code HASHTYPE} will be reported correctly.
+ * <p>
+ * This rewriter is similar to {@link ExasolJdbcQueryRewriter} but uses {@code IMPORT INTO (...) FROM EXA}.
+ */
+class ExasolFromExaWithDataTypeQueryRewriter extends AbstractQueryRewriter {
+
+    private static final Logger LOGGER = Logger.getLogger(ExasolFromExaWithDataTypeQueryRewriter.class.getName());
+    private final ConnectionFactory connectionFactory;
+
+    ExasolFromExaWithDataTypeQueryRewriter(final SqlDialect dialect, final RemoteMetadataReader remoteMetadataReader,
+            final ConnectionFactory connectionFactory) {
+        super(dialect, remoteMetadataReader, new ExasolConnectionDefinitionBuilder());
+        this.connectionFactory = connectionFactory;
+    }
+
+    @Override
+    protected String generateImportStatement(final String connectionDefinition,
+            final List<DataType> selectListDataTypes, final String pushdownQuery) throws SQLException {
+        return generateImportStatement(SqlGenerationHelper.createColumnsDescriptionFromDataTypes(selectListDataTypes),
+                connectionDefinition, pushdownQuery);
+    }
+
+    @Override
+    protected String generateImportStatement(final String connectionDefinition, final String pushdownQuery)
+            throws SQLException {
+        return generateImportStatement(createColumnsDescriptionFromQuery(pushdownQuery), connectionDefinition,
+                pushdownQuery);
+    }
+
+    private String generateImportStatement(final String columnsDescription, final String connectionDefinition,
+            final String pushdownQuery) {
+        return "IMPORT INTO (" + columnsDescription + ") FROM EXA " //
+                + connectionDefinition + " STATEMENT '" //
+                + pushdownQuery.replace("'", "''") + "'";
+    }
+
+    private String createColumnsDescriptionFromQuery(final String query) throws SQLException {
+        final ColumnMetadataReader columnMetadataReader = this.remoteMetadataReader.getColumnMetadataReader();
+        final ResultSetMetadataReader resultSetMetadataReader = new ResultSetMetadataReader(
+                this.connectionFactory.getConnection(), columnMetadataReader);
+        final String columnsDescription = resultSetMetadataReader.describeColumns(query);
+        LOGGER.finer(() -> "Import columns: " + columnsDescription);
+        return columnsDescription;
+    }
+}

src/main/java/com/exasol/adapter/dialects/exasol/ExasolProperties.java

@@ -7,8 +7,9 @@ final class ExasolProperties {
     static final String EXASOL_IMPORT_PROPERTY = "IMPORT_FROM_EXA";
     static final String EXASOL_CONNECTION_PROPERTY = "EXA_CONNECTION";
     static final String EXASOL_IS_LOCAL_PROPERTY = "IS_LOCAL";
+    static final String GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA = "GENERATE_JDBC_DATATYPE_MAPPING_FOR_EXA";
 
     private ExasolProperties() {
         // prevent instantiation
     }
-}
+}