Enabling Application Default Credentials for BigQuery (#1210)

* Enabling Application Default Credentials for BigQuery * Updated documentation, cleaned up code, and bumped versions. * Fixed documentation * Resolved lint issue. * BigQuery credentials with just a projectId are valid -- remove test.
dataform-co · Apr 22, 2021 · 42bedcb · 42bedcb
1 parent 707c87e
commit 42bedcb
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 18 deletions.
diff --git a/api/dbadapters/bigquery.ts b/api/dbadapters/bigquery.ts
@@ -316,9 +316,10 @@ export class BigQueryDbAdapter implements IDbAdapter {
         projectId,
         new BigQuery({
           projectId,
-          credentials: JSON.parse(this.bigQueryCredentials.credentials),
           scopes: EXTRA_GOOGLE_SCOPES,
-          location: this.bigQueryCredentials.location
+          location: this.bigQueryCredentials.location,
+          credentials: this.bigQueryCredentials.credentials &&
+                       JSON.parse(this.bigQueryCredentials.credentials)
         })
       );
     }

diff --git a/cli/credentials.ts b/cli/credentials.ts
@@ -5,6 +5,26 @@ import { actuallyResolve } from "df/cli/util";
 import { dataform } from "df/protos/ts";
 
 export function getBigQueryCredentials(): dataform.IBigQuery {
+  const locationIndex = selectionQuestion("Enter the location of your datasets:", [
+    "US (default)",
+    "EU",
+    "other"
+  ]);
+  let location = locationIndex === 0 ? "US" : "EU";
+  if (locationIndex === 2) {
+    location = question("Enter the location's region name (e.g. 'asia-south1'):");
+  }
+  const isApplicationDefaultOrJSONKeyIndex = selectionQuestion("Do you wish to use Application Default Credentials or JSON Key:", [
+    "ADC (default)",
+    "JSON Key"
+  ]);
+  if (isApplicationDefaultOrJSONKeyIndex === 0)  {
+    const projectId = question("Enter your billing project ID:");
+    return {
+      projectId,
+      location
+    }
+  }
   const cloudCredentialsPath = actuallyResolve(
     question(
       "Please follow the instructions at https://docs.dataform.co/dataform-cli#create-a-credentials-file/\n" +
@@ -17,15 +37,6 @@ export function getBigQueryCredentials(): dataform.IBigQuery {
     throw new Error(`Google Cloud private key file "${cloudCredentialsPath}" does not exist!`);
   }
   const cloudCredentials = JSON.parse(fs.readFileSync(cloudCredentialsPath, "utf8"));
-  const locationIndex = selectionQuestion("Enter the location of your datasets:", [
-    "US (default)",
-    "EU",
-    "other"
-  ]);
-  let location = locationIndex === 0 ? "US" : "EU";
-  if (locationIndex === 2) {
-    location = question("Enter the location's region name (e.g. 'asia-south1'):");
-  }
   return {
     projectId: cloudCredentials.project_id,
     credentials: fs.readFileSync(cloudCredentialsPath, "utf8"),

diff --git a/content/docs/getting-started-tutorial/set-up.md b/content/docs/getting-started-tutorial/set-up.md
@@ -33,7 +33,15 @@ For this this tutorial we’ll use BigQuery. Anyone with a Google Account can us
 
 ## Generate warehouse credentials
 
-In order for Dataform to connect to your BigQuery warehouse you’ll need to generate some credentials. Dataform will connect to BigQuery using a service account. You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery.
+In order for Dataform to connect to your BigQuery warehouse you’ll need to use Application Default Credentials or a service account and JSON key.
+
+### Using Application Default Credentials
+
+If running on GCE or GKE this will be automatically available. If not, then use `[gcloud auth application-default](https://cloud.google.com/sdk/gcloud/reference/auth/application-default)` to authenticate.
+
+### Create a service account with JSON key
+
+You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery.
 
 1. To create a new service account in Google Cloud Console you need to:
 

diff --git a/content/docs/warehouses/bigquery.md b/content/docs/warehouses/bigquery.md
@@ -6,10 +6,18 @@ priority: 1
 
 ## Authentication
 
-Dataform will connect to BigQuery using a service account. You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery.
+Dataform will connect to BigQuery using Application Default Credentials or using a service account.
+
+### Application Default Credentials
+
+If using Application Default Credentials ensure that the service account or user has `BigQuery Admin` role or equivalent. (Dataform requires access to create queries and list tables.) Read <a target="_blank" rel="noopener" href="https://cloud.google.com/iam/docs/granting-roles-to-service-accounts#granting_access_to_a_service_account_for_a_resource">this</a> if you need help.
+
+### Service Account
+
+You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery.
 
 1. Follow <a target="_blank" rel="noopener" href="https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating_a_service_account">these instructions</a> to create a new service account in Google Cloud Console.
-2. Grant the new account the `BigQuery Admin` role. (Admin access is required by Dataform so that it can create queries and list tables.) Read
+2. Grant the new account the `BigQuery Admin` role. (Admin access is required by Dataform so that it can create datasets and list tables.) Read
    <a target="_blank" rel="noopener" href="https://cloud.google.com/iam/docs/granting-roles-to-service-accounts#granting_access_to_a_service_account_for_a_resource">this</a> if you need help.
 3. Create a key for your new service account (in JSON format). You will upload this file to Dataform. Read
    <a target="_blank" rel="noopener" href="https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys">this</a> if you need help.

diff --git a/core/adapters/index.ts b/core/adapters/index.ts
@@ -50,8 +50,7 @@ export function supportsCancel(warehouseType: WarehouseType) {
 }
 
 const requiredBigQueryWarehouseProps: Array<keyof dataform.IBigQuery> = [
-  "projectId",
-  "credentials"
+  "projectId"
 ];
 const requiredJdbcWarehouseProps: Array<keyof dataform.IJDBC> = [
   "host",

diff --git a/protos/profiles.proto b/protos/profiles.proto
@@ -43,6 +43,7 @@ message Snowflake {
 
 message BigQuery {
   string project_id = 1;
+  // If credentials are unset, then the library will use the application default credentials
   string credentials = 3;
   // Options are listed here: https://cloud.google.com/bigquery/docs/locations
   string location = 4;

diff --git a/tests/api/api.spec.ts b/tests/api/api.spec.ts
@@ -930,7 +930,7 @@ select \${foo}
       );
     });
 
-    [{}, { wrongProperty: "" }, { projectId: "" }].forEach(bigquery => {
+    [{}, { wrongProperty: ""}].forEach(bigquery => {
       test("bigquery_properties_check", () => {
         expect(() =>
           credentials.coerce("bigquery", JSON.parse(JSON.stringify(bigquery)))

diff --git a/version.bzl b/version.bzl
@@ -1,3 +1,3 @@
 # NOTE: If you change the format of this line, you must change the bash command
 # in /scripts/publish to extract the version string correctly.
-DF_VERSION = "1.18.0"
+DF_VERSION = "1.19.0"