From 168c2ff5b761b293331d94253342245609f14d9d Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Mon, 27 Nov 2023 16:19:19 +0000 Subject: [PATCH 1/2] fix(firestore-bigquery-export): rollback backfill --- firestore-bigquery-export/CHANGELOG.md | 4 ++ firestore-bigquery-export/extension.yaml | 58 +------------------ .../functions/src/config.ts | 2 +- 3 files changed, 6 insertions(+), 58 deletions(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 373ffbda7..10d1e5b8f 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.41 + +fix - rollback backfill feature + ## Version 0.1.40 fix - correct default value for use collection group query param diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index b80951bb1..89d3ddb6e 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.1.40 +version: 0.1.41 specVersion: v1beta displayName: Stream Firestore to BigQuery @@ -338,62 +338,6 @@ params: value: no default: no required: true - - - param: DO_BACKFILL - label: Import existing Firestore documents into BigQuery? - description: >- - Do you want to import existing documents from your Firestore collection into BigQuery? These documents - will have each have a special changelog with the operation of `IMPORT` and the timestamp of epoch. - This ensures that any operation on an imported document supersedes the import record. - type: select - required: true - options: - - label: Yes - value: yes - - label: No - value: no - - - param: IMPORT_COLLECTION_PATH - label: Existing documents collection - description: >- - What is the path of the the Cloud Firestore Collection you would like to import from? - (This may, or may not, be the same Collection for which you plan to mirror changes.) - If you want to use a collectionGroup query, provide the collection name value here, - and set 'Use Collection Group query' to true. - type: string - validationRegex: "^[^/]+(/[^/]+/[^/]+)*$" - validationErrorMessage: Firestore collection paths must be an odd number of segments separated by slashes, e.g. "path/to/collection". - example: posts - required: false - - - param: USE_COLLECTION_GROUP_QUERY - label: Use Collection Group query - description: >- - Do you want to use a [collection group](https://firebase.google.com/docs/firestore/query-data/queries#collection-group-query) query for importing existing documents? - Warning: A collectionGroup query will target every collection in your Firestore project that matches the 'Existing documents collection'. - For example, if you have 10,000 documents with a sub-collection named: landmarks, this will query every document in 10,000 landmarks collections. - type: select - default: no - options: - - label: Yes - value: yes - - label: No - value: no - - - param: DOCS_PER_BACKFILL - label: Docs per backfill - description: >- - When importing existing documents, how many should be imported at once? - The default value of 200 should be ok for most users. - If you are using a transform function or have very large documents, you may need to set this to a lower number. - If the lifecycle event function times out, lower this value. - type: string - example: 200 - validationRegex: "^[1-9][0-9]*$" - validationErrorMessage: Must be a postive integer. - default: 200 - required: true - - param: KMS_KEY_NAME label: Cloud KMS key name diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts index 369f53db9..878879901 100644 --- a/firestore-bigquery-export/functions/src/config.ts +++ b/firestore-bigquery-export/functions/src/config.ts @@ -35,7 +35,7 @@ export default { bqProjectId: process.env.BIGQUERY_PROJECT_ID, collectionPath: process.env.COLLECTION_PATH, datasetId: process.env.DATASET_ID, - doBackfill: process.env.DO_BACKFILL === "yes", + doBackfill: false, docsPerBackfill: parseInt(process.env.DOCS_PER_BACKFILL) || 200, tableId: process.env.TABLE_ID, location: process.env.LOCATION, From 1718cc3781ad745a7a36eb4841a758e07dbceb7b Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 27 Nov 2023 17:16:35 +0000 Subject: [PATCH 2/2] restart ci