Skip to content

Commit 136819b

Browse files
samdentysamdenty
authored
chore(providers/openai): re-introduce logprobs as providerMetadata (#6049)
## Background In #5896 we removed logprobs to simplify models. ## Summary Re-introduce logprobs as provider-specific functionality for openai.
1 parent 9bd5ab5 commit 136819b

12 files changed

Lines changed: 476 additions & 4 deletions

‎.changeset/pink-mangos-tickle.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/openai': patch
3+
---
4+
5+
chore(providers/openai): re-introduce logprobs as providerMetadata

‎content/providers/01-ai-sdk-providers/02-openai.mdx‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,32 @@ You need to change Zod `.nullish()` and `.optional()` to `.nullable()`.
303303

304304
</Note>
305305

306+
#### Logprobs
307+
308+
OpenAI provides logprobs information for completion/chat models.
309+
You can access it in the `providerMetadata` object.
310+
311+
```ts highlight="11"
312+
import { openai } from '@ai-sdk/openai';
313+
import { generateText } from 'ai';
314+
315+
const result = await generateText({
316+
model: openai('gpt-4o'),
317+
prompt: 'Write a vegetarian lasagna recipe for 4 people.',
318+
providerOptions: {
319+
openai: {
320+
// this can also be a number,
321+
// refer to logprobs provider options section for more
322+
logprobs: true,
323+
},
324+
},
325+
});
326+
327+
const openaiMetadata = (await result.providerMetadata)?.openai;
328+
329+
const logprobs = openaiMetadata?.logprobs;
330+
```
331+
306332
#### PDF support
307333

308334
The OpenAI Chat API supports reading PDF files.

‎examples/ai-core/src/generate-text/openai-logprobs.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { openai } from '@ai-sdk/openai';
2+
import { generateText } from 'ai';
3+
import 'dotenv/config';
4+
5+
async function main() {
6+
const result = await generateText({
7+
model: openai('gpt-3.5-turbo'),
8+
prompt: 'Invent a new holiday and describe its traditions.',
9+
providerOptions: {
10+
openai: {
11+
logprobs: 2,
12+
},
13+
},
14+
});
15+
16+
console.log(result.providerMetadata?.openai.logprobs);
17+
}
18+
19+
main().catch(console.error);

‎examples/ai-core/src/stream-object/openai-fullstream.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ async function main() {
3636

3737
case 'finish': {
3838
console.log('Finish reason:', part.finishReason);
39+
console.log('Logprobs:', part.providerMetadata?.openai.logprobs);
3940
console.log('Usage:', part.usage);
4041
break;
4142
}

‎examples/ai-core/src/stream-text/azure-fullstream-logprobs.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { azure } from '@ai-sdk/azure';
2+
import { streamText } from 'ai';
3+
import 'dotenv/config';
4+
5+
async function main() {
6+
const result = streamText({
7+
model: azure('gpt-4o'),
8+
prompt: 'Invent a new holiday and describe its traditions.',
9+
providerOptions: {
10+
openai: {
11+
logprobs: 2,
12+
},
13+
},
14+
});
15+
16+
for await (const part of result.fullStream) {
17+
switch (part.type) {
18+
case 'text': {
19+
console.log('Text:', part.text);
20+
break;
21+
}
22+
23+
case 'finish': {
24+
console.log(`finishReason: ${part.finishReason}`);
25+
console.log('Logprobs:', part.providerMetadata?.azure.logprobs); // object: { string, number, array}
26+
}
27+
}
28+
}
29+
}
30+
31+
main().catch(console.error);

‎examples/ai-core/src/stream-text/openai-fullstream-logprobs.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { openai } from '@ai-sdk/openai';
2+
import { streamText } from 'ai';
3+
import 'dotenv/config';
4+
5+
async function main() {
6+
const result = streamText({
7+
model: openai('gpt-3.5-turbo'),
8+
maxOutputTokens: 512,
9+
temperature: 0.3,
10+
maxRetries: 5,
11+
prompt: 'Invent a new holiday and describe its traditions.',
12+
providerOptions: {
13+
openai: {
14+
logprobs: 2,
15+
},
16+
},
17+
});
18+
19+
for await (const part of result.fullStream) {
20+
switch (part.type) {
21+
case 'finish': {
22+
console.log('Logprobs:', part.providerMetadata?.openai.logprobs);
23+
break;
24+
}
25+
26+
case 'error':
27+
console.error('Error:', part.error);
28+
break;
29+
}
30+
}
31+
}
32+
33+
main().catch(console.error);

‎packages/openai/src/openai-chat-language-model.test.ts‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,101 @@ const TEST_PROMPT: LanguageModelV2Prompt = [
1010
{ role: 'user', content: [{ type: 'text', text: 'Hello' }] },
1111
];
1212

13+
const TEST_LOGPROBS = {
14+
content: [
15+
{
16+
token: 'Hello',
17+
logprob: -0.0009994634,
18+
top_logprobs: [
19+
{
20+
token: 'Hello',
21+
logprob: -0.0009994634,
22+
},
23+
],
24+
},
25+
{
26+
token: '!',
27+
logprob: -0.13410144,
28+
top_logprobs: [
29+
{
30+
token: '!',
31+
logprob: -0.13410144,
32+
},
33+
],
34+
},
35+
{
36+
token: ' How',
37+
logprob: -0.0009250381,
38+
top_logprobs: [
39+
{
40+
token: ' How',
41+
logprob: -0.0009250381,
42+
},
43+
],
44+
},
45+
{
46+
token: ' can',
47+
logprob: -0.047709424,
48+
top_logprobs: [
49+
{
50+
token: ' can',
51+
logprob: -0.047709424,
52+
},
53+
],
54+
},
55+
{
56+
token: ' I',
57+
logprob: -0.000009014684,
58+
top_logprobs: [
59+
{
60+
token: ' I',
61+
logprob: -0.000009014684,
62+
},
63+
],
64+
},
65+
{
66+
token: ' assist',
67+
logprob: -0.009125131,
68+
top_logprobs: [
69+
{
70+
token: ' assist',
71+
logprob: -0.009125131,
72+
},
73+
],
74+
},
75+
{
76+
token: ' you',
77+
logprob: -0.0000066306106,
78+
top_logprobs: [
79+
{
80+
token: ' you',
81+
logprob: -0.0000066306106,
82+
},
83+
],
84+
},
85+
{
86+
token: ' today',
87+
logprob: -0.00011093382,
88+
top_logprobs: [
89+
{
90+
token: ' today',
91+
logprob: -0.00011093382,
92+
},
93+
],
94+
},
95+
{
96+
token: '?',
97+
logprob: -0.00004596782,
98+
top_logprobs: [
99+
{
100+
token: '?',
101+
logprob: -0.00004596782,
102+
},
103+
],
104+
},
105+
],
106+
};
107+
13108
const provider = createOpenAI({
14109
apiKey: 'test-api-key',
15110
compatibility: 'strict',
@@ -35,6 +130,7 @@ describe('doGenerate', () => {
35130
id = 'chatcmpl-95ZTZkhr0mHNKqerQfiwkuox3PHAd',
36131
created = 1711115037,
37132
model = 'gpt-3.5-turbo-0125',
133+
logprobs = null,
38134
headers,
39135
}: {
40136
content?: string;
@@ -50,6 +146,15 @@ describe('doGenerate', () => {
50146
name: string;
51147
arguments: string;
52148
};
149+
logprobs?: {
150+
content:
151+
| {
152+
token: string;
153+
logprob: number;
154+
top_logprobs: { token: string; logprob: number }[];
155+
}[]
156+
| null;
157+
} | null;
53158
usage?: {
54159
prompt_tokens?: number;
55160
total_tokens?: number;
@@ -86,6 +191,7 @@ describe('doGenerate', () => {
86191
tool_calls,
87192
function_call,
88193
},
194+
...(logprobs ? { logprobs } : {}),
89195
finish_reason,
90196
},
91197
],
@@ -137,6 +243,7 @@ describe('doGenerate', () => {
137243
"body": {
138244
"frequency_penalty": undefined,
139245
"logit_bias": undefined,
246+
"logprobs": undefined,
140247
"max_completion_tokens": undefined,
141248
"max_tokens": undefined,
142249
"messages": [
@@ -158,6 +265,7 @@ describe('doGenerate', () => {
158265
"temperature": undefined,
159266
"tool_choice": undefined,
160267
"tools": undefined,
268+
"top_logprobs": undefined,
161269
"top_p": undefined,
162270
"user": undefined,
163271
},
@@ -224,6 +332,24 @@ describe('doGenerate', () => {
224332
expect(usage).toStrictEqual({ inputTokens: 20, outputTokens: undefined });
225333
});
226334

335+
it('should extract logprobs', async () => {
336+
prepareJsonResponse({
337+
logprobs: TEST_LOGPROBS,
338+
});
339+
340+
const response = await provider.chat('gpt-3.5-turbo').doGenerate({
341+
prompt: TEST_PROMPT,
342+
providerOptions: {
343+
openai: {
344+
logprobs: 1,
345+
},
346+
},
347+
});
348+
expect(response.providerMetadata?.openai.logprobs).toStrictEqual(
349+
TEST_LOGPROBS.content,
350+
);
351+
});
352+
227353
it('should extract finish reason', async () => {
228354
prepareJsonResponse({
229355
content: '',
@@ -1214,6 +1340,7 @@ describe('doStream', () => {
12141340
total_tokens: 244,
12151341
completion_tokens: 227,
12161342
},
1343+
logprobs = null,
12171344
finish_reason = 'stop',
12181345
model = 'gpt-3.5-turbo-0613',
12191346
headers,
@@ -1232,6 +1359,15 @@ describe('doStream', () => {
12321359
rejected_prediction_tokens?: number;
12331360
};
12341361
};
1362+
logprobs?: {
1363+
content:
1364+
| {
1365+
token: string;
1366+
logprob: number;
1367+
top_logprobs: { token: string; logprob: number }[];
1368+
}[]
1369+
| null;
1370+
} | null;
12351371
finish_reason?: string;
12361372
model?: string;
12371373
headers?: Record<string, string>;
@@ -1249,7 +1385,9 @@ describe('doStream', () => {
12491385
);
12501386
}),
12511387
`data: {"id":"chatcmpl-96aZqmeDpA9IPD6tACY8djkMsJCMP","object":"chat.completion.chunk","created":1702657020,"model":"${model}",` +
1252-
`"system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"${finish_reason}","logprobs":null}]}\n\n`,
1388+
`"system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"${finish_reason}","logprobs":${JSON.stringify(
1389+
logprobs,
1390+
)}}]}\n\n`,
12531391
`data: {"id":"chatcmpl-96aZqmeDpA9IPD6tACY8djkMsJCMP","object":"chat.completion.chunk","created":1702657020,"model":"${model}",` +
12541392
`"system_fingerprint":"fp_3bc1b5746c","choices":[],"usage":${JSON.stringify(
12551393
usage,
@@ -1268,6 +1406,7 @@ describe('doStream', () => {
12681406
total_tokens: 244,
12691407
completion_tokens: 227,
12701408
},
1409+
logprobs: TEST_LOGPROBS,
12711410
});
12721411

12731412
const { stream } = await model.doStream({
@@ -1900,6 +2039,7 @@ describe('doStream', () => {
19002039
"body": {
19012040
"frequency_penalty": undefined,
19022041
"logit_bias": undefined,
2042+
"logprobs": undefined,
19032043
"max_completion_tokens": undefined,
19042044
"max_tokens": undefined,
19052045
"messages": [
@@ -1925,6 +2065,7 @@ describe('doStream', () => {
19252065
"temperature": undefined,
19262066
"tool_choice": undefined,
19272067
"tools": undefined,
2068+
"top_logprobs": undefined,
19282069
"top_p": undefined,
19292070
"user": undefined,
19302071
},

0 commit comments

Comments
 (0)