fix(firestore-bigquery-export): fixed timestamp as a fieldname partit…

…ioning
firebase · May 15, 2024 · d8eb24f · d8eb24f
1 parent b3b844d
commit d8eb24f
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 5 deletions.
diff --git a/...tore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/e2e.test.ts b/...tore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/e2e.test.ts
@@ -267,6 +267,41 @@ describe("e2e", () => {
         );
       });
 
+      test("successfully partitions with a valid Firebase Timestamp value with a timestamp field and Timestamp type", async () => {
+        const created = firestore.Timestamp.now();
+        const expectedDate = created.toDate().toISOString().substring(0, 22);
+
+        const event: FirestoreDocumentChangeEvent = changeTrackerEvent({
+          data: { created },
+        });
+
+        await changeTracker({
+          datasetId,
+          tableId,
+          timePartitioning: "DAY",
+          timePartitioningField: "timestamp",
+          timePartitioningFieldType: "TIMESTAMP",
+          timePartitioningFirestoreField: "created",
+        }).record([event]);
+
+        const [metadata] = await dataset.table(`${tableId_raw}`).getMetadata();
+
+        const [changeLogRows] = await getBigQueryTableData(
+          process.env.PROJECT_ID,
+          datasetId,
+          tableId
+        );
+
+        expect(metadata.timePartitioning).toBeDefined();
+        expect(metadata.timePartitioning.type).toEqual("DAY");
+        expect(metadata.timePartitioning.field).toEqual("timestamp");
+
+        //TODO: check data has been added successfully
+        expect(changeLogRows[0].timestamp.value).toBe(
+          BigQuery.timestamp(created.toDate()).value
+        );
+      });
+
       test("old_data is null if is not provided", async () => {
         const event: FirestoreDocumentChangeEvent = changeTrackerEvent({
           data: { foo: "foo" },

diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts
@@ -239,41 +239,48 @@ export class Partitioning {
   }
 
   customFieldExists(fields = []) {
-    if (!fields.length) return false;
-
+    /** Extract the time partioning field name */
     const { timePartitioningField } = this.config;
 
+    /** Return based the field already exist */
     return fields.map(($) => $.name).includes(timePartitioningField);
   }
 
-  private async shouldAddPartitioningToSchema(): Promise<{
+  private async shouldAddPartitioningToSchema(fields: string[]): Promise<{
     proceed: boolean;
     message: string;
   }> {
     if (!this.isPartitioningEnabled()) {
       return { proceed: false, message: "Partitioning not enabled" };
     }
+
     if (!this.hasValidTableReference()) {
       return { proceed: false, message: "Invalid table reference" };
     }
+
     if (!this.hasValidCustomPartitionConfig()) {
       return { proceed: false, message: "Invalid partition config" };
     }
+
     if (!this.hasValidTimePartitionType()) {
       return { proceed: false, message: "Invalid partition type" };
     }
+
     if (!this.hasValidTimePartitionOption()) {
       return { proceed: false, message: "Invalid partition option" };
     }
+
     if (this.hasHourAndDatePartitionConfig()) {
       return {
         proceed: false,
         message: "Invalid partitioning and field type combination",
       };
     }
-    if (this.customFieldExists()) {
+
+    if (this.customFieldExists(fields)) {
       return { proceed: false, message: "Field already exists on schema" };
     }
+
     if (await this.isTablePartitioned()) {
       return { proceed: false, message: "Table is already partitioned" };
     }
@@ -284,7 +291,10 @@ export class Partitioning {
   }
 
   async addPartitioningToSchema(fields = []): Promise<void> {
-    const { proceed, message } = await this.shouldAddPartitioningToSchema();
+    const { proceed, message } = await this.shouldAddPartitioningToSchema(
+      fields
+    );
+
     if (!proceed) {
       functions.logger.warn(`Did not add partitioning to schema: ${message}`);
       return;