Support emulators:export and import for auth. (firebase#2955)

* Support emulators:export and import for auth. * Remove projectId from exported data. * Fix one type-o. * Export more than one user in tests. * Handle the case where there are no users. * Fix one ENOENT error for stat.
devpeerapong · Dec 18, 2020 · 9e175eb · 9e175eb
1 parent b24b972
commit 9e175eb
Show file tree

Hide file tree

Showing 7 changed files with 367 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,2 +1,3 @@
 - Improves error handling for `firestore:delete` when deleting very large documents.
 - Support batchCreate API in Auth Emulator (#2947).
+- Support emulators:export and import for Auth Emulator (#2955).
diff --git a/scripts/triggers-end-to-end-tests/tests.ts b/scripts/triggers-end-to-end-tests/tests.ts
@@ -406,4 +406,169 @@ describe("import/export end to end", () => {
     const bData = JSON.parse(fs.readFileSync(bPath).toString());
     expect(bData).to.equal(null);
   });
+
+  it("should be able to import/export auth data", async function(this) {
+    this.timeout(2 * TEST_SETUP_TIMEOUT);
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    // Start up emulator suite
+    const project = FIREBASE_PROJECT || "example";
+    const emulatorsCLI = new CLIProcess("1", __dirname);
+
+    await emulatorsCLI.start("emulators:start", project, ["--only", "auth"], (data: unknown) => {
+      if (typeof data != "string" && !Buffer.isBuffer(data)) {
+        throw new Error(`data is not a string or buffer (${typeof data})`);
+      }
+      return data.includes(ALL_EMULATORS_STARTED_LOG);
+    });
+
+    // Create some accounts to export:
+    const config = readConfig();
+    const port = config.emulators!.auth.port;
+    try {
+      process.env.FIREBASE_AUTH_EMULATOR_HOST = `localhost:${port}`;
+      const adminApp = admin.initializeApp(
+        {
+          projectId: project,
+          credential: ADMIN_CREDENTIAL,
+        },
+        "admin-app"
+      );
+      await adminApp
+        .auth()
+        .createUser({ uid: "123", email: "foo@example.com", password: "testing" });
+      await adminApp
+        .auth()
+        .createUser({ uid: "456", email: "bar@example.com", emailVerified: true });
+
+      // Ask for export
+      const exportCLI = new CLIProcess("2", __dirname);
+      const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data"));
+      await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => {
+        if (typeof data != "string" && !Buffer.isBuffer(data)) {
+          throw new Error(`data is not a string or buffer (${typeof data})`);
+        }
+        return data.includes("Export complete");
+      });
+      await exportCLI.stop();
+
+      // Stop the suite
+      await emulatorsCLI.stop();
+
+      // Confirm the data is exported as expected
+      const configPath = path.join(exportPath, "auth_export", "config.json");
+      const configData = JSON.parse(fs.readFileSync(configPath).toString());
+      expect(configData).to.deep.equal({
+        signIn: {
+          allowDuplicateEmails: false,
+        },
+      });
+
+      const accountsPath = path.join(exportPath, "auth_export", "accounts.json");
+      const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString());
+      expect(accountsData.users).to.have.length(2);
+      expect(accountsData.users[0]).to.deep.contain({
+        localId: "123",
+        email: "foo@example.com",
+        emailVerified: false,
+        providerUserInfo: [
+          {
+            email: "foo@example.com",
+            federatedId: "foo@example.com",
+            providerId: "password",
+            rawId: "foo@example.com",
+          },
+        ],
+      });
+      expect(accountsData.users[0].passwordHash).to.match(/:password=testing$/);
+      expect(accountsData.users[1]).to.deep.contain({
+        localId: "456",
+        email: "bar@example.com",
+        emailVerified: true,
+      });
+
+      // Attempt to import
+      const importCLI = new CLIProcess("3", __dirname);
+      await importCLI.start(
+        "emulators:start",
+        project,
+        ["--only", "auth", "--import", exportPath],
+        (data: unknown) => {
+          if (typeof data != "string" && !Buffer.isBuffer(data)) {
+            throw new Error(`data is not a string or buffer (${typeof data})`);
+          }
+          return data.includes(ALL_EMULATORS_STARTED_LOG);
+        }
+      );
+
+      // Check users are indeed imported correctly
+      const user1 = await adminApp.auth().getUserByEmail("foo@example.com");
+      expect(user1.passwordHash).to.match(/:password=testing$/);
+      const user2 = await adminApp.auth().getUser("456");
+      expect(user2.emailVerified).to.be.true;
+
+      await importCLI.stop();
+    } finally {
+      delete process.env.FIREBASE_AUTH_EMULATOR_HOST;
+    }
+  });
+
+  it("should be able to export / import auth data with no users", async function(this) {
+    this.timeout(2 * TEST_SETUP_TIMEOUT);
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    // Start up emulator suite
+    const project = FIREBASE_PROJECT || "example";
+    const emulatorsCLI = new CLIProcess("1", __dirname);
+
+    await emulatorsCLI.start("emulators:start", project, ["--only", "auth"], (data: unknown) => {
+      if (typeof data != "string" && !Buffer.isBuffer(data)) {
+        throw new Error(`data is not a string or buffer (${typeof data})`);
+      }
+      return data.includes(ALL_EMULATORS_STARTED_LOG);
+    });
+
+    // Ask for export (with no users)
+    const exportCLI = new CLIProcess("2", __dirname);
+    const exportPath = fs.mkdtempSync(path.join(os.tmpdir(), "emulator-data"));
+    await exportCLI.start("emulators:export", project, [exportPath], (data: unknown) => {
+      if (typeof data != "string" && !Buffer.isBuffer(data)) {
+        throw new Error(`data is not a string or buffer (${typeof data})`);
+      }
+      return data.includes("Export complete");
+    });
+    await exportCLI.stop();
+
+    // Stop the suite
+    await emulatorsCLI.stop();
+
+    // Confirm the data is exported as expected
+    const configPath = path.join(exportPath, "auth_export", "config.json");
+    const configData = JSON.parse(fs.readFileSync(configPath).toString());
+    expect(configData).to.deep.equal({
+      signIn: {
+        allowDuplicateEmails: false,
+      },
+    });
+
+    const accountsPath = path.join(exportPath, "auth_export", "accounts.json");
+    const accountsData = JSON.parse(fs.readFileSync(accountsPath).toString());
+    expect(accountsData.users).to.have.length(0);
+
+    // Attempt to import
+    const importCLI = new CLIProcess("3", __dirname);
+    await importCLI.start(
+      "emulators:start",
+      project,
+      ["--only", "auth", "--import", exportPath],
+      (data: unknown) => {
+        if (typeof data != "string" && !Buffer.isBuffer(data)) {
+          throw new Error(`data is not a string or buffer (${typeof data})`);
+        }
+        return data.includes(ALL_EMULATORS_STARTED_LOG);
+      }
+    );
+
+    await importCLI.stop();
+  });
 });
diff --git a/src/emulator/auth/index.ts b/src/emulator/auth/index.ts
@@ -1,7 +1,12 @@
+import * as fs from "fs";
+import * as path from "path";
+import * as http from "http";
 import * as utils from "../../utils";
 import { Constants } from "../constants";
+import { EmulatorLogger } from "../emulatorLogger";
 import { Emulators, EmulatorInstance, EmulatorInfo } from "../types";
 import { createApp } from "./server";
+import { FirebaseError } from "../../error";
 
 export interface AuthEmulatorArgs {
   projectId: string;
@@ -43,4 +48,116 @@ export class AuthEmulator implements EmulatorInstance {
   getName(): Emulators {
     return Emulators.AUTH;
   }
+
+  async importData(authExportDir: string, projectId: string): Promise<void> {
+    const logger = EmulatorLogger.forEmulator(Emulators.DATABASE);
+    const { host, port } = this.getInfo();
+
+    // TODO: In the future when we support import on demand, clear data first.
+
+    const configPath = path.join(authExportDir, "config.json");
+    const configStat = await stat(configPath);
+    if (configStat?.isFile()) {
+      logger.logLabeled("BULLET", "auth", `Importing config from ${configPath}`);
+
+      await importFromFile(
+        {
+          method: "PATCH",
+          host,
+          port,
+          path: `/emulator/v1/projects/${projectId}/config`,
+          headers: {
+            Authorization: "Bearer owner",
+            "Content-Type": "application/json",
+          },
+        },
+        configPath
+      );
+    }
+
+    const accountsPath = path.join(authExportDir, "accounts.json");
+    const accountsStat = await stat(accountsPath);
+    if (accountsStat?.isFile()) {
+      logger.logLabeled("BULLET", "auth", `Importing accounts from ${accountsPath}`);
+
+      await importFromFile(
+        {
+          method: "POST",
+          host,
+          port,
+          path: `/identitytoolkit.googleapis.com/v1/projects/${projectId}/accounts:batchCreate`,
+          headers: {
+            Authorization: "Bearer owner",
+            "Content-Type": "application/json",
+          },
+        },
+        accountsPath,
+        // Ignore the error when there are no users. No action needed.
+        { ignoreErrors: ["MISSING_USER_ACCOUNT"] }
+      );
+    }
+  }
+}
+
+function stat(path: fs.PathLike): Promise<fs.Stats | undefined> {
+  return new Promise((resolve, reject) =>
+    fs.stat(path, (err, stats) => {
+      if (err) {
+        if (err.code === "ENOENT") {
+          return resolve(undefined);
+        }
+        return reject(err);
+      } else {
+        return resolve(stats);
+      }
+    })
+  );
+}
+
+function importFromFile(
+  reqOptions: http.RequestOptions,
+  path: fs.PathLike,
+  options: { ignoreErrors?: string[] } = {}
+): Promise<void> {
+  const readStream = fs.createReadStream(path);
+
+  return new Promise<void>((resolve, reject) => {
+    const req = http.request(reqOptions, (response) => {
+      if (response.statusCode === 200) {
+        resolve();
+      } else {
+        let data = "";
+        response
+          .on("data", (d) => {
+            data += d.toString();
+          })
+          .on("error", reject)
+          .on("end", () => {
+            const ignoreErrors = options?.ignoreErrors;
+            if (ignoreErrors?.length) {
+              let message;
+              try {
+                message = JSON.parse(data).error.message;
+              } catch {
+                message = undefined;
+              }
+              if (message && ignoreErrors.includes(message)) {
+                return resolve();
+              }
+            }
+            return reject(
+              new FirebaseError(`Received HTTP status code: ${response.statusCode}\n${data}`)
+            );
+          });
+      }
+    });
+
+    req.on("error", reject);
+    readStream.pipe(req, { end: true });
+  }).catch((e) => {
+    throw new FirebaseError(`Error during Auth Emulator import: ${e.message}`, {
+      original: e,
+      exit: 1,
+    });
+  });
 }
diff --git a/src/emulator/auth/operations.ts b/src/emulator/auth/operations.ts
@@ -401,14 +401,15 @@ function batchGet(
   ctx: ExegesisContext
 ): Schemas["GoogleCloudIdentitytoolkitV1DownloadAccountResponse"] {
   const limit = Math.min(Math.floor(ctx.params.query.maxResults) || 20, 1000);
-  assert(limit >= 0, "((Auth Emulator: maxResults must not be negative.))");
 
   const users = state.queryUsers(
     {},
     { sortByField: "localId", order: "ASC", startToken: ctx.params.query.nextPageToken }
   );
   let newPageToken: string | undefined = undefined;
-  if (users.length > limit) {
+
+  // As a non-standard behavior, passing in limit=-1 will return all users.
+  if (limit >= 0 && users.length > limit) {
     users.length = limit;
     if (users.length) {
       newPageToken = users[users.length - 1].localId;

diff --git a/src/emulator/controller.ts b/src/emulator/controller.ts
@@ -549,6 +549,13 @@ export async function startAll(options: any, noUi: boolean = false): Promise<voi
       projectId,
     });
     await startEmulator(authEmulator);
+
+    if (exportMetadata.auth) {
+      const importDirAbsPath = path.resolve(options.import);
+      const authExportDir = path.resolve(importDirAbsPath, exportMetadata.auth.path);
+
+      await authEmulator.importData(authExportDir, projectId);
+    }
   }
 
   if (shouldStart(options, Emulators.PUBSUB)) {