Merge pull request #331 from FirebasePrivate/fz/big-query-timestamp-micro-precision

Change timestamp to a string to preserve microsecond precision

Author: fredzqm

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -87,13 +87,13 @@ export class FirestoreBigQueryEventHistoryTracker implements FirestoreEventHisto
   buildDataRow(
     eventId: string,
     changeType: ChangeType,
-    timestamp: Date,
+    timestamp: string,
     document_name: string,
     data?: Object
   ): bigquery.RowMetadata {
     // This must match firestoreToBQTable().
     return {
-      timestamp: timestamp.toISOString(),
+      timestamp: timestamp,
       event_id: eventId,
       document_name: document_name,
       operation: ChangeType[changeType],

firestore-bigquery-export/firestore-bigquery-change-tracker/src/tracker.ts

@@ -25,7 +25,10 @@ export enum ChangeType {
 }
 
 export interface FirestoreDocumentChangeEvent {
-  timestamp: Date;
+  // The timestamp represented in ISO format.
+  // Date is not appropriate because it only has millisecond precision.
+  // Cloud Firestore timestamps have microsecond precision.
+  timestamp: string;
   operation: ChangeType;
   documentName: string;
   eventId: string;

firestore-bigquery-export/functions/lib/index.js

@@ -36,7 +36,7 @@ exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change,
     try {
         const changeType = util_1.getChangeType(change);
         yield eventTracker.record([{
-                timestamp: util_1.getTimestamp(context, change),
+                timestamp: context.timestamp,
                 operation: changeType,
                 documentName: context.resource.name,
                 eventId: context.eventId,

firestore-bigquery-export/functions/lib/util.js

@@ -26,19 +26,3 @@ function getChangeType(change) {
     return firestore_bigquery_change_tracker_1.ChangeType.UPDATE;
 }
 exports.getChangeType = getChangeType;
-function getTimestamp(context, change) {
-    const changeType = getChangeType((change));
-    switch (changeType) {
-        case firestore_bigquery_change_tracker_1.ChangeType.CREATE:
-            return change.after.updateTime.toDate();
-        case firestore_bigquery_change_tracker_1.ChangeType.DELETE:
-            // Due to an internal bug (129264426), before.update_time is actually the commit timestamp.
-            return new Date(change.before.updateTime.toDate().getTime() + 1);
-        case firestore_bigquery_change_tracker_1.ChangeType.UPDATE:
-            return change.after.updateTime.toDate();
-        default: {
-            throw new Error(`Invalid change type: ${changeType}`);
-        }
-    }
-}
-exports.getTimestamp = getTimestamp;

firestore-bigquery-export/functions/src/index.ts

@@ -15,20 +15,14 @@
  */
 
 import config from "./config";
-import * as firebase from "firebase-admin";
 import * as functions from "firebase-functions";
 import {
   ChangeType,
   FirestoreBigQueryEventHistoryTracker,
   FirestoreEventHistoryTracker,
-  FirestoreSchema,
 } from "@firebaseextensions/firestore-bigquery-change-tracker";
-import * as _ from "lodash";
 import * as logs from "./logs";
-import {
-  getChangeType,
-  getTimestamp,
-} from "./util";
+import { getChangeType } from "./util";
 
 const eventTracker: FirestoreEventHistoryTracker = new FirestoreBigQueryEventHistoryTracker(config);
 
@@ -40,7 +34,7 @@ exports.fsexportbigquery = functions.handler.firestore.document.onWrite(
     try {
       const changeType = getChangeType(change);
       await eventTracker.record([{
-        timestamp: getTimestamp(context, change),
+        timestamp: context.timestamp, // This is a Cloud Firestore commit timestamp with microsecond precision.
         operation: changeType,
         documentName: context.resource.name,
         eventId: context.eventId,

firestore-bigquery-export/functions/src/util.ts

@@ -14,9 +14,8 @@
  * limitations under the License.
  */
 
-import * as firebase from "firebase-admin";
 import { DocumentSnapshot } from "firebase-functions/lib/providers/firestore";
-import { Change, EventContext } from "firebase-functions";
+import { Change } from "firebase-functions";
 
 import {
   ChangeType
@@ -31,19 +30,3 @@ export function getChangeType(change: Change<DocumentSnapshot>): ChangeType {
   }
   return ChangeType.UPDATE;
 }
-
-export function getTimestamp(context: EventContext, change: Change<DocumentSnapshot>): Date {
-  const changeType = getChangeType((change));
-  switch (changeType) {
-    case ChangeType.CREATE:
-      return change.after.updateTime.toDate();
-    case ChangeType.DELETE:
-      // Due to an internal bug (129264426), before.update_time is actually the commit timestamp.
-      return new Date(change.before.updateTime.toDate().getTime() + 1);
-    case ChangeType.UPDATE:
-      return change.after.updateTime.toDate();
-    default: {
-      throw new Error(`Invalid change type: ${changeType}`);
-    }
-  }
-}

firestore-bigquery-export/scripts/import/src/index.ts

@@ -177,7 +177,7 @@ const run = async (): Promise<number> => {
     const rows: FirestoreDocumentChangeEvent[] = docs.map(
       (snapshot) => {
         return {
-          timestamp: new Date(0), // epoch
+          timestamp: new Date(0).toISOString(), // epoch
           operation: ChangeType.IMPORT,
           documentName: snapshot.ref.path,
           eventId: "",