From 42bedcba494c8d46b9fbc6aaf40c0202d217f29e Mon Sep 17 00:00:00 2001 From: mescanne Date: Thu, 22 Apr 2021 15:52:39 +0100 Subject: [PATCH] Enabling Application Default Credentials for BigQuery (#1210) * Enabling Application Default Credentials for BigQuery * Updated documentation, cleaned up code, and bumped versions. * Fixed documentation * Resolved lint issue. * BigQuery credentials with just a projectId are valid -- remove test. --- api/dbadapters/bigquery.ts | 5 ++-- cli/credentials.ts | 29 +++++++++++++------ .../docs/getting-started-tutorial/set-up.md | 10 ++++++- content/docs/warehouses/bigquery.md | 12 ++++++-- core/adapters/index.ts | 3 +- protos/profiles.proto | 1 + tests/api/api.spec.ts | 2 +- version.bzl | 2 +- 8 files changed, 46 insertions(+), 18 deletions(-) diff --git a/api/dbadapters/bigquery.ts b/api/dbadapters/bigquery.ts index 8b94c03f7..fb91aa383 100644 --- a/api/dbadapters/bigquery.ts +++ b/api/dbadapters/bigquery.ts @@ -316,9 +316,10 @@ export class BigQueryDbAdapter implements IDbAdapter { projectId, new BigQuery({ projectId, - credentials: JSON.parse(this.bigQueryCredentials.credentials), scopes: EXTRA_GOOGLE_SCOPES, - location: this.bigQueryCredentials.location + location: this.bigQueryCredentials.location, + credentials: this.bigQueryCredentials.credentials && + JSON.parse(this.bigQueryCredentials.credentials) }) ); } diff --git a/cli/credentials.ts b/cli/credentials.ts index 3fd4f9c61..53237f2e1 100644 --- a/cli/credentials.ts +++ b/cli/credentials.ts @@ -5,6 +5,26 @@ import { actuallyResolve } from "df/cli/util"; import { dataform } from "df/protos/ts"; export function getBigQueryCredentials(): dataform.IBigQuery { + const locationIndex = selectionQuestion("Enter the location of your datasets:", [ + "US (default)", + "EU", + "other" + ]); + let location = locationIndex === 0 ? "US" : "EU"; + if (locationIndex === 2) { + location = question("Enter the location's region name (e.g. 'asia-south1'):"); + } + const isApplicationDefaultOrJSONKeyIndex = selectionQuestion("Do you wish to use Application Default Credentials or JSON Key:", [ + "ADC (default)", + "JSON Key" + ]); + if (isApplicationDefaultOrJSONKeyIndex === 0) { + const projectId = question("Enter your billing project ID:"); + return { + projectId, + location + } + } const cloudCredentialsPath = actuallyResolve( question( "Please follow the instructions at https://docs.dataform.co/dataform-cli#create-a-credentials-file/\n" + @@ -17,15 +37,6 @@ export function getBigQueryCredentials(): dataform.IBigQuery { throw new Error(`Google Cloud private key file "${cloudCredentialsPath}" does not exist!`); } const cloudCredentials = JSON.parse(fs.readFileSync(cloudCredentialsPath, "utf8")); - const locationIndex = selectionQuestion("Enter the location of your datasets:", [ - "US (default)", - "EU", - "other" - ]); - let location = locationIndex === 0 ? "US" : "EU"; - if (locationIndex === 2) { - location = question("Enter the location's region name (e.g. 'asia-south1'):"); - } return { projectId: cloudCredentials.project_id, credentials: fs.readFileSync(cloudCredentialsPath, "utf8"), diff --git a/content/docs/getting-started-tutorial/set-up.md b/content/docs/getting-started-tutorial/set-up.md index 5b8d5f7aa..a0f32217a 100644 --- a/content/docs/getting-started-tutorial/set-up.md +++ b/content/docs/getting-started-tutorial/set-up.md @@ -33,7 +33,15 @@ For this this tutorial we’ll use BigQuery. Anyone with a Google Account can us ## Generate warehouse credentials -In order for Dataform to connect to your BigQuery warehouse you’ll need to generate some credentials. Dataform will connect to BigQuery using a service account. You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery. +In order for Dataform to connect to your BigQuery warehouse you’ll need to use Application Default Credentials or a service account and JSON key. + +### Using Application Default Credentials + +If running on GCE or GKE this will be automatically available. If not, then use `[gcloud auth application-default](https://cloud.google.com/sdk/gcloud/reference/auth/application-default)` to authenticate. + +### Create a service account with JSON key + +You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery. 1. To create a new service account in Google Cloud Console you need to: diff --git a/content/docs/warehouses/bigquery.md b/content/docs/warehouses/bigquery.md index 3b8d08210..f61a62fee 100644 --- a/content/docs/warehouses/bigquery.md +++ b/content/docs/warehouses/bigquery.md @@ -6,10 +6,18 @@ priority: 1 ## Authentication -Dataform will connect to BigQuery using a service account. You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery. +Dataform will connect to BigQuery using Application Default Credentials or using a service account. + +### Application Default Credentials + +If using Application Default Credentials ensure that the service account or user has `BigQuery Admin` role or equivalent. (Dataform requires access to create queries and list tables.) Read this if you need help. + +### Service Account + +You’ll need to create a service account from your Google Cloud Console and assign it permissions to access BigQuery. 1. Follow these instructions to create a new service account in Google Cloud Console. -2. Grant the new account the `BigQuery Admin` role. (Admin access is required by Dataform so that it can create queries and list tables.) Read +2. Grant the new account the `BigQuery Admin` role. (Admin access is required by Dataform so that it can create datasets and list tables.) Read this if you need help. 3. Create a key for your new service account (in JSON format). You will upload this file to Dataform. Read this if you need help. diff --git a/core/adapters/index.ts b/core/adapters/index.ts index 77ee8dc0d..16a2e017e 100644 --- a/core/adapters/index.ts +++ b/core/adapters/index.ts @@ -50,8 +50,7 @@ export function supportsCancel(warehouseType: WarehouseType) { } const requiredBigQueryWarehouseProps: Array = [ - "projectId", - "credentials" + "projectId" ]; const requiredJdbcWarehouseProps: Array = [ "host", diff --git a/protos/profiles.proto b/protos/profiles.proto index 791861850..b3d24284e 100644 --- a/protos/profiles.proto +++ b/protos/profiles.proto @@ -43,6 +43,7 @@ message Snowflake { message BigQuery { string project_id = 1; + // If credentials are unset, then the library will use the application default credentials string credentials = 3; // Options are listed here: https://cloud.google.com/bigquery/docs/locations string location = 4; diff --git a/tests/api/api.spec.ts b/tests/api/api.spec.ts index b3fc50770..156ba8113 100644 --- a/tests/api/api.spec.ts +++ b/tests/api/api.spec.ts @@ -930,7 +930,7 @@ select \${foo} ); }); - [{}, { wrongProperty: "" }, { projectId: "" }].forEach(bigquery => { + [{}, { wrongProperty: ""}].forEach(bigquery => { test("bigquery_properties_check", () => { expect(() => credentials.coerce("bigquery", JSON.parse(JSON.stringify(bigquery))) diff --git a/version.bzl b/version.bzl index f5b99de94..66c46172d 100644 --- a/version.bzl +++ b/version.bzl @@ -1,3 +1,3 @@ # NOTE: If you change the format of this line, you must change the bash command # in /scripts/publish to extract the version string correctly. -DF_VERSION = "1.18.0" +DF_VERSION = "1.19.0"