firebase · jauntybrain · Jan 29, 2024 · Jan 10, 2024 · Jan 10, 2024 · Jan 11, 2024
diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md
@@ -154,6 +154,8 @@ essential for the script to insert data into an already partitioned table.)
 
 * Use new query syntax for snapshots: If enabled, snapshots will be generated with the new query syntax, which should be more performant, and avoid potential resource limitations.
 
+* Exclude old data payloads: If enabled, table rows will never contain old data (document snapshot before the update), which should be more performant, and avoid potential resource limitations.
+
 * Import existing Firestore documents into BigQuery?: Do you want to import existing documents from your Firestore collection into BigQuery? These documents  will have each have a special changelog with the operation of `IMPORT` and the timestamp of epoch. This ensures that any operation on an imported document supersedes the import record.
 
 * Existing Documents Collection: Specify the path of the Cloud Firestore Collection you would like to import from. This may or may not be the same Collection for which you plan to mirror changes. If you want to use a collectionGroup query, provide the collection name value here, and set 'Use Collection Group query' to true. You may use `{wildcard}` notation with an enabled collectionGroup query to match a subcollection of all documents in a collection (e.g., `chatrooms/{chatid}/posts`).

diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml
@@ -371,6 +371,14 @@ params:
     default: no
     required: true
 
+  - param: EXCLUDE_OLD_DATA
+    label: Exclude old data payloads
+    description: >-
+      If enabled, table rows will never contain old data (document snapshot
+      before the update), which should be more performant, and avoid potential
+      resource limitations.
+    type: select
+
   - param: DO_BACKFILL
     label: Import existing Firestore documents into BigQuery?
     description: >-

diff --git a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap
@@ -13,6 +13,7 @@ Object {
   "datasetLocation": undefined,
   "doBackfill": false,
   "docsPerBackfill": 200,
+  "excludeOldData": false,
   "importCollectionPath": undefined,
   "initialized": false,
   "instanceId": undefined,

diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts
@@ -55,6 +55,7 @@ export default {
   wildcardIds: process.env.WILDCARD_IDS === "true",
   useNewSnapshotQuerySyntax:
     process.env.USE_NEW_SNAPSHOT_QUERY_SYNTAX === "yes" ? true : false,
+  excludeOldData: process.env.EXCLUDE_OLD_DATA === "yes" ? true : false,
   instanceId: process.env.EXT_INSTANCE_ID!,
   maxDispatchesPerSecond: parseInt(
     process.env.MAX_DISPATCHES_PER_SECOND || "10"

diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts
@@ -110,7 +110,8 @@ export const fsexportbigquery = functions
       const isDeleted = changeType === ChangeType.DELETE;
 
       const data = isDeleted ? undefined : change.after.data();
-      const oldData = isCreated ? undefined : change.before.data();
+      const oldData =
+        isCreated || config.excludeOldData ? undefined : change.before.data();
 
       await events.recordStartEvent({
         documentId,