vercel · lgrammel · Nov 13, 2023 · Nov 8, 2023 · Nov 8, 2023 · Nov 8, 2023
diff --git a/examples/next-openai/app/api/chat-with-vision/route.ts b/examples/next-openai/app/api/chat-with-vision/route.ts
@@ -0,0 +1,44 @@
+// ./app/api/chat/route.ts
+import OpenAI from 'openai';
+import { OpenAIStream, StreamingTextResponse } from 'ai';
+
+// Create an OpenAI API client (that's edge friendly!)
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY || '',
+});
+
+// IMPORTANT! Set the runtime to edge
+export const runtime = 'edge';
+
+export async function POST(req: Request) {
+  // Extract the `prompt` from the body of the request
+  const { messages, data } = await req.json();
+
+  const initialMessages = messages.slice(0, -1);
+  const currentMessage = messages[messages.length - 1];
+
+  // Ask OpenAI for a streaming chat completion given the prompt
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4-vision-preview',
+    stream: true,
+    max_tokens: 150,
+    messages: [
+      ...initialMessages,
+      {
+        ...currentMessage,
+        content: [
+          { type: 'text', text: currentMessage.content },
+          {
+            type: 'image_url',
+            image_url: data.imageUrl,
+          },
+        ],
+      },
+    ],
+  });
+
+  // Convert the response into a friendly text-stream
+  const stream = OpenAIStream(response);
+  // Respond with the stream
+  return new StreamingTextResponse(stream);
+}
diff --git a/examples/next-openai/app/vision/page.tsx b/examples/next-openai/app/vision/page.tsx
@@ -0,0 +1,39 @@
+'use client';
+
+import { useChat } from 'ai/react';
+
+export default function Chat() {
+  const { messages, input, handleInputChange, handleSubmit, data } = useChat({
+    api: '/api/chat-with-vision',
+  });
+  return (
+    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
+      {messages.length > 0
+        ? messages.map(m => (
+            <div key={m.id} className="whitespace-pre-wrap">
+              {m.role === 'user' ? 'User: ' : 'AI: '}
+              {m.content}
+            </div>
+          ))
+        : null}
+
+      <form
+        onSubmit={e => {
+          handleSubmit(e, {
+            data: {
+              imageUrl:
+                'https://upload.wikimedia.org/wikipedia/commons/thumb/3/3c/Field_sparrow_in_CP_%2841484%29_%28cropped%29.jpg/733px-Field_sparrow_in_CP_%2841484%29_%28cropped%29.jpg',
+            },
+          });
+        }}
+      >
+        <input
+          className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl"
+          value={input}
+          placeholder="What does the image show..."
+          onChange={handleInputChange}
+        />
+      </form>
+    </div>
+  );
+}
diff --git a/packages/core/react/use-chat.ts b/packages/core/react/use-chat.ts
@@ -75,6 +75,7 @@ export type UseChatHelpers = {
 
 type StreamingReactResponseAction = (payload: {
   messages: Message[];
+  data?: Record<string, string>;
 }) => Promise<experimental_StreamingReactResponse>;
 
 const getStreamedResponse = async (
@@ -135,6 +136,7 @@ const getStreamedResponse = async (
     try {
       const promise = api({
         messages: constructedMessagesPayload as Message[],
+        data: chatRequest.data,
       }) as Promise<ReactResponseRow>;
       await readRow(promise);
     } catch (e) {
@@ -154,6 +156,7 @@ const getStreamedResponse = async (
     method: 'POST',
     body: JSON.stringify({
       messages: constructedMessagesPayload,
+      data: chatRequest.data,
       ...extraMetadataRef.current.body,
       ...chatRequest.options?.body,
       ...(chatRequest.functions !== undefined && {
@@ -478,7 +481,7 @@ export function useChat({
   const append = useCallback(
     async (
       message: Message | CreateMessage,
-      { options, functions, function_call }: ChatRequestOptions = {},
+      { options, functions, function_call, data }: ChatRequestOptions = {},
     ) => {
       if (!message.id) {
         message.id = nanoid();
@@ -487,6 +490,7 @@ export function useChat({
       const chatRequest: ChatRequest = {
         messages: messagesRef.current.concat(message as Message),
         options,
+        data,
         ...(functions !== undefined && { functions }),
         ...(function_call !== undefined && { function_call }),
       };
@@ -546,7 +550,7 @@ export function useChat({
   const handleSubmit = useCallback(
     (
       e: React.FormEvent<HTMLFormElement>,
-      { options, functions, function_call }: ChatRequestOptions = {},
+      options: ChatRequestOptions = {},
       metadata?: Object,
     ) => {
       if (metadata) {
@@ -565,7 +569,7 @@ export function useChat({
           role: 'user',
           createdAt: new Date(),
         },
-        { options, functions, function_call },
+        options,
       );
       setInput('');
     },

diff --git a/packages/core/shared/types.ts b/packages/core/shared/types.ts
@@ -71,6 +71,7 @@ export type ChatRequest = {
   options?: RequestOptions;
   functions?: Array<Function>;
   function_call?: FunctionCall;
+  data?: Record<string, string>;
 };
 
 export type FunctionCallHandler = (
@@ -87,6 +88,7 @@ export type ChatRequestOptions = {
   options?: RequestOptions;
   functions?: Array<Function>;
   function_call?: FunctionCall;
+  data?: Record<string, string>;
 };
 
 export type UseChatOptions = {