Always export any imported namespaces (firebase#2648)

devpeerapong · Oct 8, 2020 · 226ea56 · 226ea56
1 parent 8f80feb
commit 226ea56
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1 @@
+- Fixes issue where database export does not work if database is empty (#2634).
diff --git a/scripts/triggers-end-to-end-tests/tests.ts b/scripts/triggers-end-to-end-tests/tests.ts
@@ -347,7 +347,7 @@ describe("import/export end to end", () => {
     await importCLI.start(
       "emulators:start",
       FIREBASE_PROJECT,
-      ["--only", "database", "--import", exportPath],
+      ["--only", "database", "--import", exportPath, "--export-on-exit"],
       (data: unknown) => {
         if (typeof data != "string" && !Buffer.isBuffer(data)) {
           throw new Error(`data is not a string or buffer (${typeof data})`);
@@ -362,6 +362,28 @@ describe("import/export end to end", () => {
     expect(aSnap.val()).to.eql("namespace-a");
     expect(bSnap.val()).to.eql("namespace-b");
 
+    // Delete all of the import files
+    for (const f of fs.readdirSync(dbExportPath)) {
+      const fullPath = path.join(dbExportPath, f);
+      await fs.unlinkSync(fullPath);
+    }
+
+    // Delete all the data in one namespace
+    await bApp
+      .database()
+      .ref()
+      .set(null);
+
+    // Stop the CLI (which will export on exit)
     await importCLI.stop();
+
+    // Confirm the data exported is as expected
+    const aPath = path.join(dbExportPath, "namespace-a.json");
+    const aData = JSON.parse(fs.readFileSync(aPath).toString());
+    expect(aData).to.deep.equal({ ns: "namespace-a" });
+
+    const bPath = path.join(dbExportPath, "namespace-b.json");
+    const bData = JSON.parse(fs.readFileSync(bPath).toString());
+    expect(bData).to.equal(null);
   });
 });
diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts
@@ -2,7 +2,6 @@ import * as _ from "lodash";
 import * as clc from "cli-color";
 import * as fs from "fs";
 import * as path from "path";
-import * as http from "http";
 
 import * as Config from "../config";
 import * as logger from "../logger";
@@ -528,42 +527,7 @@ export async function startAll(options: any, noUi: boolean = false): Promise<voi
       for (const f of files) {
         const fPath = path.join(databaseExportDir, f);
         const ns = path.basename(f, ".json");
-
-        databaseLogger.logLabeled("BULLET", "database", `Importing data from ${fPath}`);
-
-        const readStream = fs.createReadStream(fPath);
-
-        await new Promise((resolve, reject) => {
-          const req = http.request(
-            {
-              method: "PUT",
-              host: databaseAddr.host,
-              port: databaseAddr.port,
-              path: `/.json?ns=${ns}&disableTriggers=true&writeSizeLimit=unlimited`,
-              headers: {
-                Authorization: "Bearer owner",
-                "Content-Type": "application/json",
-              },
-            },
-            (response) => {
-              if (response.statusCode === 200) {
-                resolve();
-              } else {
-                databaseLogger.log("DEBUG", "Database import failed: " + response.statusCode);
-                response
-                  .on("data", (d) => {
-                    databaseLogger.log("DEBUG", d.toString());
-                  })
-                  .on("end", reject);
-              }
-            }
-          );
-
-          req.on("error", reject);
-          readStream.pipe(req, { end: true });
-        }).catch((e) => {
-          throw new FirebaseError("Error during database import.", { original: e, exit: 1 });
-        });
+        await databaseEmulator.importData(ns, fPath);
       }
     }
   }

diff --git a/src/emulator/databaseEmulator.ts b/src/emulator/databaseEmulator.ts
@@ -2,13 +2,15 @@ import * as chokidar from "chokidar";
 import * as clc from "cli-color";
 import * as fs from "fs";
 import * as path from "path";
+import * as http from "http";
 
 import * as api from "../api";
 import * as downloadableEmulators from "./downloadableEmulators";
 import { EmulatorInfo, EmulatorInstance, Emulators } from "../emulator/types";
 import { Constants } from "./constants";
 import { EmulatorRegistry } from "./registry";
 import { EmulatorLogger } from "./emulatorLogger";
+import { FirebaseError } from "../error";
 
 export interface DatabaseEmulatorArgs {
   port?: number;
@@ -21,6 +23,7 @@ export interface DatabaseEmulatorArgs {
 }
 
 export class DatabaseEmulator implements EmulatorInstance {
+  private importedNamespaces: string[] = [];
   private rulesWatcher?: chokidar.FSWatcher;
   private logger = EmulatorLogger.forEmulator(Emulators.DATABASE);
 
@@ -106,6 +109,50 @@ export class DatabaseEmulator implements EmulatorInstance {
     return Emulators.DATABASE;
   }
 
+  getImportedNamespaces(): string[] {
+    return this.importedNamespaces;
+  }
+
+  async importData(ns: string, fPath: string): Promise<void> {
+    this.logger.logLabeled("BULLET", "database", `Importing data from ${fPath}`);
+
+    const readStream = fs.createReadStream(fPath);
+    const { host, port } = this.getInfo();
+
+    await new Promise((resolve, reject) => {
+      const req = http.request(
+        {
+          method: "PUT",
+          host,
+          port,
+          path: `/.json?ns=${ns}&disableTriggers=true&writeSizeLimit=unlimited`,
+          headers: {
+            Authorization: "Bearer owner",
+            "Content-Type": "application/json",
+          },
+        },
+        (response) => {
+          if (response.statusCode === 200) {
+            this.importedNamespaces.push(ns);
+            resolve();
+          } else {
+            this.logger.log("DEBUG", "Database import failed: " + response.statusCode);
+            response
+              .on("data", (d) => {
+                this.logger.log("DEBUG", d.toString());
+              })
+              .on("end", reject);
+          }
+        }
+      );
+
+      req.on("error", reject);
+      readStream.pipe(req, { end: true });
+    }).catch((e) => {
+      throw new FirebaseError("Error during database import.", { original: e, exit: 1 });
+    });
+  }
+
   private async updateRules(instance: string, content: string): Promise<any> {
     const { host, port } = this.getInfo();
     try {

diff --git a/src/emulator/hubExport.ts b/src/emulator/hubExport.ts
@@ -9,6 +9,7 @@ import { EmulatorRegistry } from "./registry";
 import { FirebaseError } from "../error";
 import { EmulatorHub } from "./hub";
 import { getDownloadDetails } from "./downloadableEmulators";
+import { DatabaseEmulator } from "./databaseEmulator";
 
 export interface FirestoreExportMetadata {
   version: string;
@@ -92,29 +93,38 @@ export class HubExport {
   }
 
   private async exportDatabase(metadata: ExportMetadata): Promise<void> {
-    const databaseInfo = EmulatorRegistry.get(Emulators.DATABASE)!.getInfo();
-    const databaseAddr = `http://${databaseInfo.host}:${databaseInfo.port}`;
+    const databaseEmulator = EmulatorRegistry.get(Emulators.DATABASE) as DatabaseEmulator;
+    const { host, port } = databaseEmulator.getInfo();
+    const databaseAddr = `http://${host}:${port}`;
 
     // Get the list of namespaces
     const inspectURL = `/.inspect/databases.json?ns=${this.projectId}`;
     const inspectRes = await api.request("GET", inspectURL, { origin: databaseAddr, auth: true });
     const namespaces = inspectRes.body.map((instance: any) => instance.name);
 
     // Check each one for actual data
-    const nonEmptyNamespaces = [];
+    const namespacesToExport = [];
     for (const ns of namespaces) {
       const checkDataPath = `/.json?ns=${ns}&shallow=true&limitToFirst=1`;
       const checkDataRes = await api.request("GET", checkDataPath, {
         origin: databaseAddr,
         auth: true,
       });
       if (checkDataRes.body !== null) {
-        nonEmptyNamespaces.push(ns);
+        namespacesToExport.push(ns);
       } else {
         logger.debug(`Namespace ${ns} contained null data, not exporting`);
       }
     }
 
+    // We always need to export every namespace that was imported
+    for (const ns of databaseEmulator.getImportedNamespaces()) {
+      if (!namespacesToExport.includes(ns)) {
+        logger.debug(`Namespace ${ns} was imported, exporting.`);
+        namespacesToExport.push(ns);
+      }
+    }
+
     // Make sure the export directory exists
     if (!fs.existsSync(this.exportPath)) {
       fs.mkdirSync(this.exportPath);
@@ -125,7 +135,7 @@ export class HubExport {
       fs.mkdirSync(dbExportPath);
     }
 
-    for (const ns of nonEmptyNamespaces) {
+    for (const ns of namespacesToExport) {
       const exportFile = path.join(dbExportPath, `${ns}.json`);
       const writeStream = fs.createWriteStream(exportFile);
 
@@ -134,8 +144,8 @@ export class HubExport {
         http
           .get(
             {
-              host: databaseInfo.host,
-              port: databaseInfo.port,
+              host,
+              port,
               path: `/.json?ns=${ns}&format=export`,
               headers: { Authorization: "Bearer owner" },
             },