feat(generative-ai): Add basic samples for Gemini / VertexAI inference (

#3670) * feat(generative-ai): Add sample for basic text generation. * chore: fix lint issues * feat: Add basic multimodal example * chore: add header * feat: Add streaming inference example * feat: Add example for multimodal streaming response * fix: Fix some tests that were not running correctly * chore: Clarify test names * fix: Adjust cloud storage bucket and argurment order for consistency * chore: fix lint errors * chore: Update some tests and prompt text to be clearer * Update model to the correct multimodal model * Clarify multimodal prompt based on order * chore: Update function calling stream model to address flaky test.
GoogleCloudPlatform · May 6, 2024 · f377daa · f377daa
1 parent dcfe8fa
commit f377daa
Show file tree

Hide file tree

Showing 11 changed files with 415 additions and 4 deletions.
diff --git a/generative-ai/snippets/inference/nonStreamMultiModalityBasic.js b/generative-ai/snippets/inference/nonStreamMultiModalityBasic.js
@@ -0,0 +1,63 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_non_stream_multimodality_basic]
+const {VertexAI} = require('@google-cloud/vertexai');
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function generateContent(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.5-pro-preview-0409'
+) {
+  // Initialize Vertex AI
+  const vertexAI = new VertexAI({project: projectId, location: location});
+  const generativeModel = vertexAI.getGenerativeModel({model: model});
+
+  const request = {
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {
+            file_data: {
+              file_uri: 'gs://cloud-samples-data/video/animals.mp4',
+              mime_type: 'video/mp4',
+            },
+          },
+          {
+            file_data: {
+              file_uri:
+                'gs://cloud-samples-data/generative-ai/image/character.jpg',
+              mime_type: 'image/jpeg',
+            },
+          },
+          {text: 'Are this video and image correlated?'},
+        ],
+      },
+    ],
+  };
+
+  const result = await generativeModel.generateContent(request);
+
+  console.log(result.response.candidates[0].content.parts[0].text);
+}
+// [END generativeaionvertexai_non_stream_multimodality_basic]
+
+generateContent(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/inference/nonStreamTextBasic.js b/generative-ai/snippets/inference/nonStreamTextBasic.js
@@ -0,0 +1,58 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_non_stream_text_basic]
+const {VertexAI} = require('@google-cloud/vertexai');
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function generateContent(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.0-pro'
+) {
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({project: projectId, location: location});
+
+  // Instantiate the model
+  const generativeModel = vertexAI.getGenerativeModel({
+    model: model,
+  });
+
+  const request = {
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {
+            text: 'Write a story about a magic backpack.',
+          },
+        ],
+      },
+    ],
+  };
+
+  console.log(JSON.stringify(request));
+
+  const result = await generativeModel.generateContent(request);
+
+  console.log(result.response.candidates[0].content.parts[0].text);
+}
+// [END generativeaionvertexai_non_stream_text_basic]
+
+generateContent(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/inference/streamMultiModalityBasic.js b/generative-ai/snippets/inference/streamMultiModalityBasic.js
@@ -0,0 +1,65 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_stream_multimodality_basic]
+const {VertexAI} = require('@google-cloud/vertexai');
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function generateContent(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.5-pro-preview-0409'
+) {
+  // Initialize Vertex AI
+  const vertexAI = new VertexAI({project: projectId, location: location});
+  const generativeModel = vertexAI.getGenerativeModel({model: model});
+
+  const request = {
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {
+            file_data: {
+              file_uri: 'gs://cloud-samples-data/video/animals.mp4',
+              mime_type: 'video/mp4',
+            },
+          },
+          {
+            file_data: {
+              file_uri:
+                'gs://cloud-samples-data/generative-ai/image/character.jpg',
+              mime_type: 'image/jpeg',
+            },
+          },
+          {text: 'Are this video and image correlated?'},
+        ],
+      },
+    ],
+  };
+
+  const result = await generativeModel.generateContentStream(request);
+
+  for await (const item of result.stream) {
+    console.log(item.candidates[0].content.parts[0].text);
+  }
+}
+// [END generativeaionvertexai_stream_multimodality_basic]
+
+generateContent(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/inference/streamTextBasic.js b/generative-ai/snippets/inference/streamTextBasic.js
@@ -0,0 +1,59 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START generativeaionvertexai_stream_text_basic]
+const {VertexAI} = require('@google-cloud/vertexai');
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function generateContent(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-1.0-pro'
+) {
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({project: projectId, location: location});
+
+  // Instantiate the model
+  const generativeModel = vertexAI.getGenerativeModel({
+    model: model,
+  });
+
+  const request = {
+    contents: [
+      {
+        role: 'user',
+        parts: [
+          {
+            text: 'Write a story about a magic backpack.',
+          },
+        ],
+      },
+    ],
+  };
+
+  console.log(JSON.stringify(request));
+
+  const result = await generativeModel.generateContentStream(request);
+  for await (const item of result.stream) {
+    console.log(item.candidates[0].content.parts[0].text);
+  }
+}
+// [END generativeaionvertexai_stream_text_basic]
+
+generateContent(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/package.json b/generative-ai/snippets/package.json
@@ -10,7 +10,7 @@
     "*.js"
   ],
   "scripts": {
-    "test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js"
+    "test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js test/**/*.test.js"
   },
   "dependencies": {
     "@google-cloud/aiplatform": "^3.12.0",

diff --git a/generative-ai/snippets/test/functionCallingStreamChat.test.js b/generative-ai/snippets/test/functionCallingStreamChat.test.js
@@ -21,7 +21,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
 
 const projectId = process.env.CAIP_PROJECT_ID;
 const location = process.env.LOCATION;
-const model = 'gemini-1.0-pro';
+const model = 'gemini-1.5-pro-preview-0409';
 
 describe('Generative AI Function Calling Stream Chat', () => {
   /**
@@ -30,7 +30,7 @@ describe('Generative AI Function Calling Stream Chat', () => {
    */
   // const projectId = 'YOUR_PROJECT_ID';
   // const location = 'YOUR_LOCATION';
-  // const model = 'gemini-1.0-pro';
+  // const model = 'gemini-1.5-pro-preview-0409';
 
   it('should create stream chat and begin the conversation the same in each instance', async () => {
     const output = execSync(

diff --git a/generative-ai/snippets/test/gemini-audio-summarization.test.js b/generative-ai/snippets/test/gemini-audio-summarization.test.js
@@ -22,7 +22,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
 const projectId = process.env.CAIP_PROJECT_ID;
 
 describe('Summarize audio', async () => {
-  it('should summerize audio', async () => {
+  it('should summarize audio', async () => {
     const output = execSync(
       `node ./gemini-audio-summarization.js ${projectId}`
     );

diff --git a/generative-ai/snippets/test/inference/nonStreamMultiModalityBasic.test.js b/generative-ai/snippets/test/inference/nonStreamMultiModalityBasic.test.js
@@ -0,0 +1,41 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const location = process.env.LOCATION;
+const model = 'gemini-1.5-pro-preview-0409';
+
+describe('Generative AI Multimodal Text Inference', () => {
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const projectId = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_LOCATION';
+  // const model = 'gemini-1.5-pro-preview-0409';
+
+  it('should generate text based on a prompt containing text, a video, and an image', async () => {
+    const output = execSync(
+      `node ./inference/nonStreamMultiModalityBasic.js ${projectId} ${location} ${model}`
+    );
+    assert(output.length > 0);
+  });
+});
diff --git a/generative-ai/snippets/test/inference/nonStreamTextBasic.test.js b/generative-ai/snippets/test/inference/nonStreamTextBasic.test.js
@@ -0,0 +1,43 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const location = process.env.LOCATION;
+const model = 'gemini-1.0-pro';
+
+describe('Generative AI Basic Text Inference', () => {
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const projectId = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_LOCATION';
+  // const model = 'gemini-1.0-pro';
+
+  it('should create a generative text model and infer text from a prompt', async () => {
+    const output = execSync(
+      `node ./inference/nonStreamTextBasic.js ${projectId} ${location} ${model}`
+    );
+
+    // Assert that the correct prompt was issued
+    assert(output.match(/Write a story about a magic backpack/));
+  });
+});