fix: log data and old data on fail to enqueue

firebase · Sep 18, 2024 · fdaaef3 · fdaaef3
1 parent 09ec773
commit fdaaef3
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 37 deletions.
diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts
@@ -97,52 +97,79 @@ export const fsexportbigquery = functions
   .document(config.collectionPath)
   .onWrite(async (change, context) => {
     logs.start();
-    try {
-      const changeType = getChangeType(change);
-      const documentId = getDocumentId(change);
 
-      const isCreated = changeType === ChangeType.CREATE;
-      const isDeleted = changeType === ChangeType.DELETE;
+    /**
+     * enqueue data cannot currently handle documentdata
+     * Serialize early before queueing in clopud task
+     * Cloud tasks currently have a limit of 1mb, this also ensures payloads are kept to a minimum
+     */
+    const changeType = getChangeType(change);
+    const documentId = getDocumentId(change);
 
-      const data = isDeleted ? undefined : change.after.data();
-      const oldData =
-        isCreated || config.excludeOldData ? undefined : change.before.data();
+    const isCreated = changeType === ChangeType.CREATE;
+    const isDeleted = changeType === ChangeType.DELETE;
 
-      await events.recordStartEvent({
-        documentId,
-        changeType,
-        before: {
-          data: change.before.data(),
-        },
-        after: {
-          data: change.after.data(),
-        },
-        context: context.resource,
-      });
+    const data = isDeleted ? undefined : change.after.data();
+    const oldData =
+      isCreated || config.excludeOldData ? undefined : change.before.data();
+
+    let serializedData: any;
+    let serializedOldData: any;
+
+    try {
+      serializedData = eventTracker.serializeData(data);
+    } catch (err) {
+      functions.logger.error("failed to serialize data", err);
+      throw err;
+    }
+    try {
+      serializedOldData = eventTracker.serializeData(oldData);
+    } catch (err) {
+      functions.logger.error("failed to serialize old data", err);
+      throw err;
+    }
+
+    try {
+      await events
+        .recordStartEvent({
+          documentId,
+          changeType,
+          before: {
+            data: change.before.data(),
+          },
+          after: {
+            data: change.after.data(),
+          },
+          context: context.resource,
+        })
+        .catch((err) => {
+          functions.logger.error("failed to record start event", err);
+
+          throw err;
+        });
 
       const queue = getFunctions().taskQueue(
         `locations/${config.location}/functions/syncBigQuery`,
         config.instanceId
       );
 
-      /**
-       * enqueue data cannot currently handle documentdata
-       * Serialize early before queueing in clopud task
-       * Cloud tasks currently have a limit of 1mb, this also ensures payloads are kept to a minimum
-       */
-      const seializedData = eventTracker.serializeData(data);
-      const serializedOldData = eventTracker.serializeData(oldData);
-
-      await queue.enqueue({
-        context,
-        changeType,
-        documentId,
-        data: seializedData,
-        oldData: serializedOldData,
-      });
+      await queue
+        .enqueue({
+          context,
+          changeType,
+          documentId,
+          data: serializedData,
+          oldData: serializedOldData,
+        })
+        .catch((err) => {
+          functions.logger.error("failed to enqueue task", err);
+
+          throw err;
+        });
     } catch (err) {
       await events.recordErrorEvent(err as Error);
-      logs.error(err);
+
+      logs.error(err, serializedData, serializedOldData);
       const eventAgeMs = Date.now() - Date.parse(context.timestamp);
       const eventMaxAgeMs = 10000;
 

diff --git a/firestore-bigquery-export/functions/src/logs.ts b/firestore-bigquery-export/functions/src/logs.ts
@@ -149,8 +149,12 @@ export const dataTypeInvalid = (
   );
 };
 
-export const error = (err: Error) => {
-  logger.error("Error when mirroring data to BigQuery", err);
+export const error = (err: Error, data, oldData) => {
+  logger.error("Error when mirroring data to BigQuery", {
+    error: err,
+    data,
+    oldData,
+  });
 };
 
 export const init = () => {