We follow the same naming convention as the corresponding system variable + */ + private String datasetProjectId = null; + + /** The ProjectId to use for billing. */ + private final String projectId; /** Boolean to determine if the Connection is closed */ private boolean isclosed = false; @@ -152,7 +169,7 @@ public BQConnection(String url, Properties loginProp, HttpTransport httpTranspor if (matchData.find()) { this.projectId = CatalogName.toProjectId(matchData.group(1)); - this.dataset = matchData.group(2); + configureDataSet(matchData.group(2)); } else { this.projectId = CatalogName.toProjectId(pathParams); } @@ -415,6 +432,19 @@ public void close() throws SQLException { } } + /** + * Parses the input dataset specification and sets the values for the dataset and datasetProjectId + * instance variables. + */ + private void configureDataSet(String datasetSpec) { + DatasetReference datasetRef = parseDatasetRef(datasetSpec, this.projectId); + this.dataset = datasetRef.getDatasetId(); + this.datasetProjectId = datasetRef.getProjectId(); + } + + /** + * Returns the default dataset that should be configured on queries processed by this connection. + */ public String getDataSet() { return this.dataset; } @@ -577,7 +607,7 @@ public Struct createStruct(String typeName, Object[] attributes) throws SQLExcep @Override public void setSchema(String schema) { - this.dataset = schema; + configureDataSet(schema); } @Override @@ -694,11 +724,19 @@ public DatabaseMetaData getMetaData() throws SQLException { return metadata; } - /** Getter method for projectId */ + /** Getter method for the projectId to use for billing. */ public String getProjectId() { return projectId; } + /** + * Returns the dataset project id that should be configured on queries processed by this + * connection. + */ + public String getDataSetProjectId() { + return this.datasetProjectId; + } + /** * * @@ -1258,4 +1296,50 @@ public Integer getTimeoutMs() { public JobCreationMode getJobCreationMode() { return jobCreationMode; } + + /** + * Returns a DatasetReference extracted from the input dataset specification, which may optionally + * include a project id reference. + * + *
While there are well-defined rules regarding project + * ids and dataset + * names, we must support historical references to both that don't adhere to the documented + * rules (such as having a domain name with a colon as part of the project id). The only + * restriction that we impose is that either identifier consists of at least one character. We + * permit using either a "." or ":" as the delimiter between the dataset id and the project id. If + * the {@code datasetSpec} is either null or an empty string, we return a DatasetReference with a + * null dataset id and its project id set to the {@code defaultProjectId} argument. + * + *
Visible for testing. + * + * @param datasetSpec Dataset specification, typically from the BQJDBC connection string. + * @param defaultProjectId Project id to set on the resulting DatasetReference if no value found + * in {@code datasetSpec}. + * @return DatasetReference + */ + static DatasetReference parseDatasetRef(@Nullable String datasetSpec, String defaultProjectId) { + if (defaultProjectId == null || defaultProjectId.isEmpty()) { + throw new IllegalArgumentException("defaultProjectId cannot be null or empty"); + } + if (datasetSpec == null || datasetSpec.isEmpty()) { + return new DatasetReference().setDatasetId(null).setProjectId(defaultProjectId); + } + // We split datasetSpec on the last occurrence of a project delimiter. To detect invalid empty + // identifiers, we pass -1 as the limit value to disable discarding empty strings. + String[] datasetComponents = datasetSpec.split(LAST_PROJECT_DELIMITER_REGEX, -1); + boolean isDatasetIdOnly = datasetComponents.length == 1; + String datasetId = isDatasetIdOnly ? datasetComponents[0] : datasetComponents[1]; + String datasetProjectId = + isDatasetIdOnly ? defaultProjectId : CatalogName.toProjectId(datasetComponents[0]); + if (datasetId.isEmpty()) { + throw new IllegalArgumentException( + String.format("Resulting dataset id is empty after parsing '%s'", datasetSpec)); + } + if (datasetProjectId.isEmpty()) { + throw new IllegalArgumentException( + String.format("Resulting dataset project id is empty after parsing '%s'", datasetSpec)); + } + return new DatasetReference().setDatasetId(datasetId).setProjectId(datasetProjectId); + } } diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java b/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java index a191caa4..82e88b2a 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java @@ -236,6 +236,7 @@ public ResultSet executeQuery() throws SQLException { this.projectId, this.RunnableStatement, this.connection.getDataSet(), + this.connection.getDataSetProjectId(), this.connection.getUseLegacySql(), this.connection.getMaxBillingBytes()); this.logger.info("Executing Query: " + this.RunnableStatement); diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java b/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java index 6310a993..6224c830 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java @@ -341,6 +341,7 @@ protected QueryResponse runSyncQuery(String querySql, boolean unlimitedBillingBy projectId, querySql, connection.getDataSet(), + connection.getDataSetProjectId(), this.connection.getUseLegacySql(), !unlimitedBillingBytes ? this.connection.getMaxBillingBytes() : null, getSyncTimeoutMillis(), // we need this to respond fast enough to avoid any diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java b/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java index bf8d08d1..a83a0156 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java @@ -260,6 +260,7 @@ private int executeDML(String sql) throws SQLException { projectId, sql, connection.getDataSet(), + connection.getDataSetProjectId(), this.connection.getUseLegacySql(), this.connection.getMaxBillingBytes(), (long) querytimeout * 1000, @@ -323,6 +324,7 @@ public ResultSet executeQuery(String querySql, boolean unlimitedBillingBytes) projectId, querySql, connection.getDataSet(), + connection.getDataSetProjectId(), this.connection.getUseLegacySql(), billingBytes, (long) querytimeout * 1000, diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java b/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java index c17d12b7..002c9592 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java @@ -68,11 +68,13 @@ public class BQSupportFuncts { */ public static String constructUrlFromPropertiesFile( Properties properties, boolean full, String dataset) throws UnsupportedEncodingException { - String projectId = properties.getProperty("projectid"); + String projectId = properties.getProperty("projectid"); // Represents the billing project. logger.debug("projectId is: " + projectId); String User = properties.getProperty("user"); String Password = properties.getProperty("password"); String path = properties.getProperty("path"); + // The dataset property value can optionally include a reference to a project id, which will be + // used in conjunction with the default dataset to handle unqualified table references. dataset = dataset == null ? properties.getProperty("dataset") : dataset; String forreturn = ""; @@ -621,10 +623,12 @@ public static Properties readFromPropFile(String filePath) throws IOException { * Run a query using the synchronous jobs.query() BigQuery endpoint. * * @param bigquery The BigQuery API wrapper - * @param projectId + * @param projectId The ProjectId to use for billing * @param querySql The SQL to execute * @param dataSet default dataset, can be null - * @param useLegacySql + * @param dataSetProjectId default dataset project id, only specified when the default dataset is + * non-null + * @param useLegacySql Use the legacy SQL dialect when true * @param maxBillingBytes Maximum bytes that the API will allow to bill * @param queryTimeoutMs The timeout at which point the API will return with an incomplete result * NOTE: this does _not_ mean the query fails, just we have to get the results async @@ -640,6 +644,7 @@ static QueryResponse runSyncQuery( String projectId, String querySql, String dataSet, + String dataSetProjectId, Boolean useLegacySql, Long maxBillingBytes, Long queryTimeoutMs, @@ -653,6 +658,7 @@ static QueryResponse runSyncQuery( projectId, querySql, dataSet, + dataSetProjectId, useLegacySql, maxBillingBytes, queryTimeoutMs, @@ -672,6 +678,7 @@ static Bigquery.Jobs.Query getSyncQuery( String projectId, String querySql, String dataSet, + String dataSetProjectId, Boolean useLegacySql, Long maxBillingBytes, Long queryTimeoutMs, @@ -692,7 +699,8 @@ static Bigquery.Jobs.Query getSyncQuery( qr = qr.setJobCreationMode(jobCreationMode.name()); } if (dataSet != null) { - qr.setDefaultDataset(new DatasetReference().setDatasetId(dataSet).setProjectId(projectId)); + qr.setDefaultDataset( + new DatasetReference().setDatasetId(dataSet).setProjectId(dataSetProjectId)); } if (maxResults != null) { qr.setMaxResults(maxResults); @@ -701,12 +709,44 @@ static Bigquery.Jobs.Query getSyncQuery( return bigquery.jobs().query(projectId, qr); } + /** + * Starts a new query in async mode. + * + *
This method exists to maintain backwards compatibility with prior bqjdbc releases. + * + * @param bigquery The bigquery instance, which is authorized + * @param projectId The project ID to use for both the billing and default dataset project ids + * @param querySql The sql query which we want to run + * @param dataSet The default dataset, can be null + * @param useLegacySql Use the legacy SQL dialect when true + * @param maxBillingBytes Maximum bytes that the API will allow to bill + * @return A JobReference which we'll use to poll the bigquery, for its state, then for its mined + * data. + * @throws IOException + *
if the request for initializing or executing job fails + */ + public static Job startQuery( + Bigquery bigquery, + String projectId, + String querySql, + String dataSet, + Boolean useLegacySql, + Long maxBillingBytes) + throws IOException { + return startQuery( + bigquery, projectId, querySql, dataSet, projectId, useLegacySql, maxBillingBytes); + } + /** * Starts a new query in async mode. * * @param bigquery The bigquery instance, which is authorized - * @param projectId The project's ID + * @param projectId The project ID to use for billing * @param querySql The sql query which we want to run + * @param dataSet The default dataset, can be null + * @param dataSetProjectId The default dataset project id, only specified when the default dataset + * is non-null + * @param useLegacySql Use the legacy SQL dialect when true * @return A JobReference which we'll use to poll the bigquery, for its state, then for its mined * data. * @throws IOException @@ -717,6 +757,7 @@ public static Job startQuery( String projectId, String querySql, String dataSet, + String dataSetProjectId, Boolean useLegacySql, Long maxBillingBytes) throws IOException { @@ -732,7 +773,7 @@ public static Job startQuery( if (dataSet != null) queryConfig.setDefaultDataset( - new DatasetReference().setDatasetId(dataSet).setProjectId(projectId)); + new DatasetReference().setDatasetId(dataSet).setProjectId(dataSetProjectId)); job.setConfiguration(config); queryConfig.setQuery(querySql); diff --git a/src/test/java/net/starschema/clouddb/jdbc/JdbcUrlTest.java b/src/test/java/net/starschema/clouddb/jdbc/JdbcUrlTest.java index d9653505..3c3f97a5 100644 --- a/src/test/java/net/starschema/clouddb/jdbc/JdbcUrlTest.java +++ b/src/test/java/net/starschema/clouddb/jdbc/JdbcUrlTest.java @@ -4,13 +4,18 @@ import com.google.api.client.testing.http.MockLowLevelHttpRequest; import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.api.services.bigquery.Bigquery.Jobs.Query; +import com.google.api.services.bigquery.model.DatasetReference; import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.security.GeneralSecurityException; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Map; import java.util.Properties; import junit.framework.Assert; @@ -36,16 +41,179 @@ public class JdbcUrlTest { public void setup() throws SQLException, IOException { properties = getProperties("/installedaccount.properties"); URL = getUrl("/installedaccount.properties", null) + "&useLegacySql=true"; - ; + this.bq = new BQConnection(URL, new Properties()); this.environmentVariables.set("GOOGLE_APPLICATION_CREDENTIALS", defaultServiceAccount); } + @Test + public void parseDatasetRefShouldWork() throws IOException { + DatasetReference datasetRef1 = + BQConnection.parseDatasetRef("project1.dataset1", "billingProject"); + DatasetReference datasetRef2 = + BQConnection.parseDatasetRef("project2:dataset2", "billingProject"); + DatasetReference datasetRef3 = BQConnection.parseDatasetRef("dataset3", "billingProject3"); + DatasetReference datasetRef4 = BQConnection.parseDatasetRef(null, "billingProject4"); + DatasetReference datasetRef5 = BQConnection.parseDatasetRef("", "billingProject5"); + DatasetReference datasetRef6 = + BQConnection.parseDatasetRef("domain:project6:dataset6", "billingProject"); + DatasetReference datasetRef7 = + BQConnection.parseDatasetRef("domain:project7.dataset7", "billingProject"); + DatasetReference datasetRef8 = + BQConnection.parseDatasetRef("domain.com:project8.data-set8", "billingProject"); + + Assert.assertEquals( + new DatasetReference().setProjectId("project1").setDatasetId("dataset1"), datasetRef1); + Assert.assertEquals( + new DatasetReference().setProjectId("project2").setDatasetId("dataset2"), datasetRef2); + Assert.assertEquals( + new DatasetReference().setProjectId("billingProject3").setDatasetId("dataset3"), + datasetRef3); + Assert.assertEquals( + new DatasetReference().setProjectId("billingProject4").setDatasetId(null), datasetRef4); + Assert.assertEquals( + new DatasetReference().setProjectId("billingProject5").setDatasetId(null), datasetRef5); + Assert.assertEquals( + new DatasetReference().setProjectId("domain:project6").setDatasetId("dataset6"), + datasetRef6); + Assert.assertEquals( + new DatasetReference().setProjectId("domain:project7").setDatasetId("dataset7"), + datasetRef7); + Assert.assertEquals( + new DatasetReference().setProjectId("domain.com:project8").setDatasetId("data-set8"), + datasetRef8); + } + + @Test + public void parseDatasetRefShouldNotWorkOnMalformedInput() { + try { + BQConnection.parseDatasetRef("project1.", "billingProject"); + + Assert.fail("Expected IllegalArgumentException to be thrown on empty dataset"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("Resulting dataset id is empty")); + } + + try { + BQConnection.parseDatasetRef("project2:", "billingProject"); + + Assert.fail("Expected IllegalArgumentException to be thrown on empty dataset"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("Resulting dataset id is empty")); + } + + try { + BQConnection.parseDatasetRef(".dataset3", "billingProject"); + + Assert.fail("Expected IllegalArgumentException to be thrown on empty project"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("Resulting dataset project id is empty")); + } + + try { + BQConnection.parseDatasetRef(":dataset4", "billingProject"); + + Assert.fail("Expected IllegalArgumentException to be thrown on empty project"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("Resulting dataset project id is empty")); + } + + try { + BQConnection.parseDatasetRef("dataset5", ""); + + Assert.fail("Expected IllegalArgumentException to be thrown on empty default project id"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("defaultProjectId cannot be null or empty")); + } + + try { + BQConnection.parseDatasetRef("dataset6", null); + + Assert.fail("Expected IllegalArgumentException to be thrown on null default dataset id"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("defaultProjectId cannot be null or empty")); + } + } + + @Test + public void constructUrlFromPropertiesFileShouldWorkWhenProjectIdAndDatasetProjectIdAreSet() + throws IOException, SQLException { + properties.put("projectid", "disco-parsec-659"); + properties.put("dataset", "publicdata.samples"); + File tempFile = File.createTempFile("tmp_installedaccount", ".properties"); + tempFile.deleteOnExit(); + try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { + properties.store(outputStream, "Test connection properties"); + } + Properties customProperties = BQSupportFuncts.readFromPropFile(tempFile.getAbsolutePath()); + + String url = BQSupportFuncts.constructUrlFromPropertiesFile(customProperties, true, null); + BQConnection connection = new BQConnection(url, new Properties()); + + Assert.assertEquals("disco-parsec-659", connection.getProjectId()); + Assert.assertEquals("publicdata", connection.getDataSetProjectId()); + Assert.assertEquals("samples", connection.getDataSet()); + } + + @Test + public void constructUrlFromPropertiesFileShouldWorkWhenOnlyProjectIdSet() + throws IOException, SQLException { + properties.put("projectid", "disco-parsec-659"); + properties.put("dataset", properties.getProperty("dataset")); + File tempFile = File.createTempFile("tmp_installedaccount", ".properties"); + tempFile.deleteOnExit(); + try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { + properties.store(outputStream, "Test connection properties"); + } + Properties customProperties = BQSupportFuncts.readFromPropFile(tempFile.getAbsolutePath()); + + String url = BQSupportFuncts.constructUrlFromPropertiesFile(customProperties, true, null); + BQConnection connection = new BQConnection(url, new Properties()); + + Assert.assertEquals("disco-parsec-659", connection.getProjectId()); + Assert.assertEquals("disco-parsec-659", connection.getDataSetProjectId()); + } + @Test public void urlWithDefaultDatasetShouldWork() throws SQLException { Assert.assertEquals(properties.getProperty("dataset"), bq.getDataSet()); } + @Test + public void urlWithoutDatasetProjectIdShouldWorkAndGetDataSetProjectIdShouldReturnProjectId() { + Assert.assertEquals("disco-parsec-659", bq.getDataSetProjectId()); + } + + @Test + public void urlWithoutDatasetShouldWorkAndGetDataSetProjectIdShouldReturnProjectId() { + String urlWithoutDataSet = URL.replace("/" + properties.getProperty("dataset"), ""); + try { + BQConnection bqWithoutDataSet = new BQConnection(urlWithoutDataSet, new Properties()); + + Assert.assertEquals("disco-parsec-659", bqWithoutDataSet.getDataSetProjectId()); + Assert.assertEquals(null, bqWithoutDataSet.getDataSet()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + @Test + public void urlWithDataSetProjectIdShouldWorkAndBeReturnedByGetDataSetProjectId() { + String urlWithDataSetProjectId = + URL.replace( + "/" + properties.getProperty("dataset"), + "/looker-test-db." + properties.getProperty("dataset")); + try { + BQConnection bqWithDataSetProjectId = + new BQConnection(urlWithDataSetProjectId, new Properties()); + + Assert.assertEquals("disco-parsec-659", bqWithDataSetProjectId.getProjectId()); + Assert.assertEquals("looker-test-db", bqWithDataSetProjectId.getDataSetProjectId()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + @Test public void projectWithColons() throws SQLException { String urlWithColonContainingProject = URL.replace(bq.getProjectId(), "example.com:project"); @@ -72,6 +240,53 @@ public void mungedProjectName() throws SQLException { } } + @Test + public void mungedDatasetProjectName() throws SQLException { + String urlWithDatasetProjectIdContainingUnderscores = + URL.replace( + "/" + properties.getProperty("dataset"), + "/example_com__project." + properties.getProperty("dataset")); + try { + BQConnection bqWithUnderscores = + new BQConnection(urlWithDatasetProjectIdContainingUnderscores, new Properties()); + Assert.assertEquals("example.com:project", bqWithUnderscores.getDataSetProjectId()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + @Test + public void setSchemaWorksWhenDataSetProjectIdIsUnspecified() throws SQLException { + this.bq.setSchema("tokyo_star"); + + Assert.assertEquals("tokyo_star", this.bq.getDataSet()); + Assert.assertEquals("disco-parsec-659", this.bq.getDataSetProjectId()); + } + + @Test + public void setSchemaWorksWhenDataSetProjectIdIsSpecified() { + this.bq.setSchema("publicdata.samples"); + + Assert.assertEquals("samples", this.bq.getDataSet()); + Assert.assertEquals("publicdata", this.bq.getDataSetProjectId()); + } + + @Test + public void setSchemaWorksWhenSchemaIsEmpty() { + this.bq.setSchema(""); + + Assert.assertEquals(null, this.bq.getDataSet()); + Assert.assertEquals("disco-parsec-659", this.bq.getDataSetProjectId()); + } + + @Test + public void setSchemaWorksWhenSchemaIsNull() { + this.bq.setSchema(null); + + Assert.assertEquals(null, this.bq.getDataSet()); + Assert.assertEquals("disco-parsec-659", this.bq.getDataSetProjectId()); + } + @Test public void urlWithTimeouts() throws SQLException { try { @@ -117,6 +332,123 @@ public void canRunQueryWithDefaultDataset() throws SQLException { stmt.executeQuery("SELECT * FROM orders limit 1"); } + @Test + public void canNotRunQueryOnStatementWithoutDatasetProjectId() throws SQLException, IOException { + // The query should fail since the billing project 'disco-parsec-659' does not have the dataset. + // 'samples'. + properties.put("dataset", "samples"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + Statement stmt = bqConnection.createStatement(); + + try { + stmt.executeQuery("SELECT * FROM shakespeare LIMIT 1"); + + Assert.fail("Expected IllegalArgumentException to be thrown without default dataset"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage().contains("not found in")); + } + } + + @Test + public void canNotRunQueryOnStatementWithoutDataset() throws SQLException, IOException { + // The query should fail since there will be no default dataset specified on the query. + properties.remove("dataset"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + Statement stmt = bqConnection.createStatement(); + + try { + stmt.executeQuery("SELECT * FROM shakespeare LIMIT 1"); + + Assert.fail("Expected IllegalArgumentException to be thrown without default dataset"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage().contains("must be qualified with a dataset")); + } + } + + @Test + public void canRunQueryOnStatementUsingDatasetProjectId() throws SQLException, IOException { + // Add the dataset project id that houses the 'samples' dataset. We do not have the ability to + // create BigQuery jobs in 'publicdata', but we can read from the 'samples' dataset in that + // project. If we fail to respect the dataset project id, then the query will fail, since the + // billing project 'disco-parsec-659' does not have that dataset. + properties.put("dataset", "publicdata.samples"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + Statement stmt = bqConnection.createStatement(); + + // This should succeed. We use the default dataset and project for the read query, but we charge + // the computation to 'disco-parsec-659'. + stmt.executeQuery("SELECT * FROM shakespeare LIMIT 1"); + } + + @Test + public void canNotRunQueryOnPreparedStatementWithoutDatasetProjectId() + throws SQLException, IOException { + // The query should fail since the billing project 'disco-parsec-659' does not have the dataset. + // 'samples'. + properties.put("dataset", "samples"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + PreparedStatement stmt = + bqConnection.prepareStatement("SELECT TOP(word, 3), COUNT(*) FROM shakespeare"); + try { + stmt.executeQuery(); + + Assert.fail("Expected IllegalArgumentException to be thrown without default dataset"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage().contains("not found in")); + } + } + + @Test + public void canNotRunQueryOnPreparedStatementWithoutDataset() throws SQLException, IOException { + // The query should fail since there will be no default dataset specified on the query. + properties.remove("dataset"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + PreparedStatement stmt = + bqConnection.prepareStatement("SELECT TOP(word, 3), COUNT(*) FROM shakespeare"); + try { + stmt.executeQuery(); + + Assert.fail("Expected IllegalArgumentException to be thrown without default dataset"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage().contains("must be qualified with a dataset")); + } + } + + @Test + public void canRunQueryOnPreparedStatementUsingDatasetProjectId() + throws SQLException, IOException { + // Add the dataset project id that houses the 'samples' dataset. We do not have the ability to + // create BigQuery jobs in 'publicdata', but we can read from the 'samples' dataset in that + // project. If we fail to respect the dataset project id, then the query will fail, since the + // billing project 'disco-parsec-659' does not have that dataset. + properties.put("dataset", "publicdata.samples"); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(properties, true, null) + + "&useLegacySql=true"; + BQConnection bqConnection = new BQConnection(url, new Properties()); + PreparedStatement stmt = + bqConnection.prepareStatement("SELECT TOP(word, 3), COUNT(*) FROM shakespeare"); + + // This should succeed. We use the default dataset and project for the read query, but we charge + // the computation to 'disco-parsec-659'. + stmt.executeQuery(); + } + @Test public void canConnectWithPasswordProtectedP12File() throws SQLException, IOException { String url = getUrl("/protectedaccount.properties", null); @@ -208,6 +540,7 @@ public void oAuthAccessTokenOnlyInHeader() oauthProps.getProperty("projectid"), "SELECT * FROM orders limit 1", bqConn.getDataSet(), + bqConn.getDataSetProjectId(), bqConn.getUseLegacySql(), null, stmt.getSyncTimeoutMillis(), diff --git a/src/test/java/net/starschema/clouddb/jdbc/PreparedStatementTests.java b/src/test/java/net/starschema/clouddb/jdbc/PreparedStatementTests.java index b8fcb142..81930162 100644 --- a/src/test/java/net/starschema/clouddb/jdbc/PreparedStatementTests.java +++ b/src/test/java/net/starschema/clouddb/jdbc/PreparedStatementTests.java @@ -21,15 +21,18 @@ package net.starschema.clouddb.jdbc; import java.io.StringReader; +import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Properties; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -41,10 +44,12 @@ * @author Balazs Gunics */ public class PreparedStatementTests { - /** Static reference to the connection object */ static Connection con = null; + /** Properties used to create connection object */ + private Properties connectionProperties; + /** * Compares two String[][] * @@ -71,12 +76,14 @@ private boolean comparer(String[][] expected, String[][] reality) { @Before public void Connect() throws Exception { try { + connectionProperties = + BQSupportFuncts.readFromPropFile( + getClass().getResource("/installedaccount1.properties").getFile()); + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(connectionProperties) + + "&useLegacySql=true"; Class.forName("net.starschema.clouddb.jdbc.BQDriver"); - PreparedStatementTests.con = - DriverManager.getConnection( - BQSupportFuncts.constructUrlFromPropertiesFile( - BQSupportFuncts.readFromPropFile("installedaccount1.properties")), - BQSupportFuncts.readFromPropFile("installedaccount1.properties")); + PreparedStatementTests.con = DriverManager.getConnection(url, connectionProperties); } catch (Exception e) { e.printStackTrace(); } @@ -146,11 +153,7 @@ public void ResultSetMetadataFunctionTestTypes() { {"SELECT CAST('2021-04-09T20:24:39' AS DATETIME)", "2021-04-09T20:24:39"}, {"SELECT CAST('1:23:45' AS TIME)", "01:23:45"}, {"SELECT CAST('test' AS BYTES)", "dGVzdA=="}, - {"SELECT CAST('123' as BIGNUMERIC)", "123"}, - { - "SELECT ST_GEOGFROMTEXT('LINESTRING(6.2312655 51.9967517, 6.2312606 51.9968043)')", - "LINESTRING(6.2312655 51.9967517, 6.2312606 51.9968043)" - } + {"SELECT CAST('123' as BIGNUMERIC)", "123"} }; final int[] expectedType = @@ -165,8 +168,7 @@ public void ResultSetMetadataFunctionTestTypes() { java.sql.Types.TIMESTAMP, java.sql.Types.TIME, java.sql.Types.VARCHAR, - java.sql.Types.NUMERIC, - java.sql.Types.VARCHAR + java.sql.Types.NUMERIC }; for (int i = 0; i < queries.length; i++) { @@ -203,6 +205,33 @@ public void ResultSetMetadataFunctionTestTypes() { con = null; } + @Test + public void ResultSetMetadataFunctionTestTypesOnQueryThatRequiresGoogleSQL() + throws SQLException, UnsupportedEncodingException { + String query = + "SELECT ST_GEOGFROMTEXT('LINESTRING(6.2312655 51.9967517, 6.2312606 51.9968043)')"; + String url = + BQSupportFuncts.constructUrlFromPropertiesFile(connectionProperties) + + "&useLegacySql=false"; + int expectedTye = Types.VARCHAR; + String expectedResult = "LINESTRING(6.2312655 51.9967517, 6.2312606 51.9968043)"; + Connection connection = DriverManager.getConnection(url, connectionProperties); + PreparedStatement stm = connection.prepareStatement(query); + + java.sql.ResultSet result = stm.executeQuery(); + + Assert.assertNotNull(result); + Assert.assertEquals( + "Expected type was not returned in metadata", + expectedTye, + result.getMetaData().getColumnType(1)); + while (result.next()) { + Assert.assertNotNull(result.getObject(1)); + Assert.assertEquals(expectedResult, result.getString(1)); + } + connection.close(); + } + /** setBigDecimal test */ @Test public void setBigDecimalTest() {