chore(*): release-2023-July-23

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,11 @@
+## Version 0.1.35
+
+fixed - add missing locations back in
+
+fixed - use module instead namespace
+
+fixed - added e2e testing, upgraded dependencies
+
 ## Version 0.1.34
 
 feat - added failure policy

firestore-bigquery-export/PREINSTALL.md

@@ -47,6 +47,33 @@ Prior to sending the document change to BigQuery, you have an opportunity to tra
 
 The response should be indentical in structure.
 
+#### Using Customer Managed Encryption Keys
+
+By default, BigQuery encrypts your content stored at rest. BigQuery handles and manages this default encryption for you without any additional actions on your part.
+
+If you want to control encryption yourself, you can use customer-managed encryption keys (CMEK) for BigQuery. Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS.
+
+For more general information on this, see [the docs](https://cloud.google.com/bigquery/docs/customer-managed-encryption).
+
+To use CMEK and the Key Management Service (KMS) with this extension
+1. [Enable the KMS API in your Google Cloud Project](https://console.cloud.google.com/apis/enableflow?apiid=cloudkms.googleapis.com).
+2. Create a keyring and keychain in the KMS. Note that the region of the keyring and key *must* match the region of your bigquery dataset
+3. Grant the BigQuery service account permission to encrypt and decrypt using that key. The Cloud KMS CryptoKey Encrypter/Decrypter role grants this permission. First find your project number. You can find this for example on the cloud console dashboard `https://console.cloud.google.com/home/dashboard?project={PROJECT_ID}`. The service account which needs the Encrypter/Decrypter role is then `[email protected]`. You can grant this role through the credentials service in the console, or through the CLI:
+```
+gcloud kms keys add-iam-policy-binding \
+--project=KMS_PROJECT_ID \
+--member serviceAccount:[email protected] \
+--role roles/cloudkms.cryptoKeyEncrypterDecrypter \
+--location=KMS_KEY_LOCATION \
+--keyring=KMS_KEY_RING \
+KMS_KEY
+```
+4. When installing this extension, enter the resource name of your key. It will look something like the following:
+```
+projects/<YOUR PROJECT ID>/locations/<YOUR REGION>/keyRings/<YOUR KEY RING NAME>/cryptoKeys/<YOUR KEY NAME>
+```
+If you follow these steps, your changelog table should be created using your customer-managed encryption.
+
 #### Backfill your BigQuery dataset
 
 This extension only sends the content of documents that have been changed -- it does not export your full dataset of existing documents into BigQuery. So, to backfill your BigQuery dataset with all the documents in your collection, you can run the [import script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md) provided by this extension.

firestore-bigquery-export/README.md

@@ -55,6 +55,33 @@ Prior to sending the document change to BigQuery, you have an opportunity to tra
 
 The response should be indentical in structure.
 
+#### Using Customer Managed Encryption Keys
+
+By default, BigQuery encrypts your content stored at rest. BigQuery handles and manages this default encryption for you without any additional actions on your part.
+
+If you want to control encryption yourself, you can use customer-managed encryption keys (CMEK) for BigQuery. Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS.
+
+For more general information on this, see [the docs](https://cloud.google.com/bigquery/docs/customer-managed-encryption).
+
+To use CMEK and the Key Management Service (KMS) with this extension
+1. [Enable the KMS API in your Google Cloud Project](https://console.cloud.google.com/apis/enableflow?apiid=cloudkms.googleapis.com).
+2. Create a keyring and keychain in the KMS. Note that the region of the keyring and key *must* match the region of your bigquery dataset
+3. Grant the BigQuery service account permission to encrypt and decrypt using that key. The Cloud KMS CryptoKey Encrypter/Decrypter role grants this permission. First find your project number. You can find this for example on the cloud console dashboard `https://console.cloud.google.com/home/dashboard?project={PROJECT_ID}`. The service account which needs the Encrypter/Decrypter role is then `[email protected]`. You can grant this role through the credentials service in the console, or through the CLI:
+```
+gcloud kms keys add-iam-policy-binding \
+--project=KMS_PROJECT_ID \
+--member serviceAccount:[email protected] \
+--role roles/cloudkms.cryptoKeyEncrypterDecrypter \
+--location=KMS_KEY_LOCATION \
+--keyring=KMS_KEY_RING \
+KMS_KEY
+```
+4. When installing this extension, enter the resource name of your key. It will look something like the following:
+```
+projects/<YOUR PROJECT ID>/locations/<YOUR REGION>/keyRings/<YOUR KEY RING NAME>/cryptoKeys/<YOUR KEY NAME>
+```
+If you follow these steps, your changelog table should be created using your customer-managed encryption.
+
 #### Backfill your BigQuery dataset
 
 This extension only sends the content of documents that have been changed -- it does not export your full dataset of existing documents into BigQuery. So, to backfill your BigQuery dataset with all the documents in your collection, you can run the [import script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md) provided by this extension.
@@ -77,8 +104,6 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan
 
 **Configuration Parameters:**
 
-* Cloud Functions location: Where do you want to deploy the functions created for this extension?  You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations).
-
 * BigQuery Dataset location: Where do you want to deploy the BigQuery dataset created for this extension? For help selecting a location, refer to the [location selection guide](https://cloud.google.com/bigquery/docs/locations).
 
 * Project Id: Override the default project bigquery instance. This can allow updates to be directed to a bigquery instance on another project.
@@ -102,18 +127,26 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan
 
 * BigQuery SQL table clustering: This parameter will allow you to set up Clustering for the BigQuery Table created by the extension. (for example: `data,document_id,timestamp`- no whitespaces). You can select up to 4 comma separated fields. The order of the specified columns determines the sort order of the data. Available schema extensions table fields for clustering: `document_id, timestamp, event_id, operation, data`.
 
+* Maximum number of synced documents per second: This parameter will set the maximum number of syncronised documents per second with BQ. Please note, any other external updates to a Big Query table will be included within this quota. Ensure that you have a set a low enough number to componsate. Defaults to 10.
+
 * Backup Collection Name: This (optional) parameter will allow you to specify a collection for which failed BigQuery updates will be written to.
 
 * Transform function URL: Specify a function URL to call that will transform the payload that will be written to BigQuery. See the pre-install documentation for more details.
 
 * Use new query syntax for snapshots: If enabled, snapshots will be generated with the new query syntax, which should be more performant, and avoid potential resource limitations.
 
+* Cloud KMS key name: Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS. If this parameter is set, the extension will specify the KMS key name when creating the BQ table. See the PREINSTALL.md for more details.
+
 
 
 **Cloud Functions:**
 
 * **fsexportbigquery:** Listens for document changes in your specified Cloud Firestore collection, then exports the changes into BigQuery.
 
+* **syncBigQuery:** A task-triggered function that gets called on BigQuery sync
+
+* **setupBigQuerySync:** Runs configuration for sycning with BigQuery
+
 
 
 **APIs Used**:

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.34
+version: 0.1.35
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery
@@ -322,6 +322,16 @@ params:
     default: no
     required: true
 
+  - param: KMS_KEY_NAME
+    label: Cloud KMS key name
+    description: >-
+      Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS.
+      If this parameter is set, the extension will specify the KMS key name when creating the BQ table. See the PREINSTALL.md for more details.
+    type: string
+    validationRegex: 'projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)'
+    validationErrorMessage: The key name must be of the format 'projects/PROJECT_NAME/locations/KEY_RING_LOCATION/keyRings/KEY_RING_ID/cryptoKeys/KEY_ID'.
+    required: false
+
 events:
   - type: firebase.extensions.big-query-export.v1.sync.start
     description: Occurs on a firestore document write event.

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.26",
+  "version": "1.1.27",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -60,6 +60,7 @@ export interface FirestoreBigQueryEventHistoryTrackerConfig {
   backupTableId?: string | undefined;
   useNewSnapshotQuerySyntax?: boolean;
   skipInit?: boolean;
+  kmsKeyName?: string | undefined;
 }
 
 /**
@@ -389,6 +390,12 @@ export class FirestoreBigQueryEventHistoryTracker
       }
       const options: TableMetadata = { friendlyName: changelogName, schema };
 
+      if (this.config.kmsKeyName) {
+        options["encryptionConfiguration"] = {
+          kmsKeyName: this.config.kmsKeyName,
+        };
+      }
+
       //Add partitioning
       await partitioning.addPartitioningToSchema(schema.fields);
 

firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -13,6 +13,7 @@ Object {
   "datasetLocation": undefined,
   "initialized": false,
   "instanceId": undefined,
+  "kmsKeyName": "test",
   "location": "us-central1",
   "maxDispatchesPerSecond": 10,
   "tableId": "my_table",

firestore-bigquery-export/functions/__tests__/config.test.ts

@@ -17,8 +17,10 @@ const environment = {
   TABLE_ID: "my_table",
   TRANSFORM_FUNCTION: "",
   CLUSTERING: "data,timestamp",
+  KMS_KEY_NAME: "test",
 };
 
+//@ts-ignore
 const { config } = global;
 
 describe("extension config", () => {
@@ -46,8 +48,8 @@ describe("extension config", () => {
       datasetId: environment.DATASET_ID,
       tableId: environment.TABLE_ID,
       clustering: clustering(environment.CLUSTERING),
+      kmsKeyName: environment.KMS_KEY_NAME,
     };
-
     expect(config()).toMatchSnapshot(env);
   });
 

firestore-bigquery-export/functions/package.json

@@ -13,7 +13,7 @@
   "author": "Jan Wyszynski <[email protected]>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.26",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.27",
     "@google-cloud/bigquery": "^4.7.0",
     "@types/chai": "^4.1.6",
     "@types/express-serve-static-core": "4.17.30",

firestore-bigquery-export/functions/src/config.ts

@@ -56,4 +56,5 @@ export default {
   maxDispatchesPerSecond: parseInt(
     process.env.MAX_DISPATCHES_PER_SECOND || "10"
   ),
+  kmsKeyName: process.env.KMS_KEY_NAME,
 };

firestore-bigquery-export/functions/src/index.ts

@@ -45,6 +45,7 @@ const eventTracker: FirestoreEventHistoryTracker =
     bqProjectId: config.bqProjectId,
     useNewSnapshotQuerySyntax: config.useNewSnapshotQuerySyntax,
     skipInit: true,
+    kmsKeyName: config.kmsKeyName,
   });
 
 logs.init();