Skip to content

Commit f20ba77

Browse files
vercel-ai-sdk[bot]R-Tanejafelixarntz
authored
Backport: feat(@ai-sdk/google): preserve per-modality token details in usage data (#14110)
This is an automated backport of #14016 to the release-v6.0 branch. FYI @R-Taneja Co-authored-by: Rohan Taneja <47066511+R-Taneja@users.noreply.github.com> Co-authored-by: Felix Arntz <felix.arntz@vercel.com>
1 parent c9ae9d5 commit f20ba77

File tree

6 files changed

+208
-0
lines changed

6 files changed

+208
-0
lines changed

‎.changeset/google-modality-token-details.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/google': patch
3+
---
4+
5+
feat(provider/google): preserve per-modality token details in usage data

‎examples/ai-functions/src/generate-text/google/modality-token-details.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { google } from '@ai-sdk/google';
2+
import { generateText } from 'ai';
3+
import { run } from '../../lib/run';
4+
5+
run(async () => {
6+
const result = await generateText({
7+
model: google('gemini-2.5-flash'),
8+
messages: [
9+
{
10+
role: 'user',
11+
content: [
12+
{ type: 'text', text: 'Describe the image in detail.' },
13+
{
14+
type: 'image',
15+
image:
16+
'https://github.com/vercel/ai/blob/main/examples/ai-functions/data/comic-cat.png?raw=true',
17+
},
18+
],
19+
},
20+
],
21+
});
22+
23+
const usageMetadata = result.providerMetadata?.google?.usageMetadata as
24+
| Record<string, unknown>
25+
| undefined;
26+
27+
console.log(result.text);
28+
console.log();
29+
console.log('Token usage:', result.usage);
30+
console.log('Modality token details:', {
31+
promptTokensDetails: usageMetadata?.promptTokensDetails,
32+
candidatesTokensDetails: usageMetadata?.candidatesTokensDetails,
33+
});
34+
});

‎examples/ai-functions/src/stream-text/google/modality-token-details.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import { google } from '@ai-sdk/google';
2+
import { streamText } from 'ai';
3+
import { run } from '../../lib/run';
4+
5+
run(async () => {
6+
const result = streamText({
7+
model: google('gemini-2.5-flash'),
8+
messages: [
9+
{
10+
role: 'user',
11+
content: [
12+
{ type: 'text', text: 'Describe the image in detail.' },
13+
{
14+
type: 'image',
15+
image:
16+
'https://github.com/vercel/ai/blob/main/examples/ai-functions/data/comic-cat.png?raw=true',
17+
},
18+
],
19+
},
20+
],
21+
});
22+
23+
for await (const textPart of result.textStream) {
24+
process.stdout.write(textPart);
25+
}
26+
27+
const usageMetadata = (await result.providerMetadata)?.google
28+
?.usageMetadata as Record<string, unknown> | undefined;
29+
30+
console.log();
31+
console.log('Token usage:', await result.usage);
32+
console.log('Modality token details:', {
33+
promptTokensDetails: usageMetadata?.promptTokensDetails,
34+
candidatesTokensDetails: usageMetadata?.candidatesTokensDetails,
35+
});
36+
});

‎packages/google/src/__snapshots__/google-generative-ai-language-model.test.ts.snap‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
3030
"usageMetadata": {
3131
"candidatesTokenCount": 29,
3232
"promptTokenCount": 9,
33+
"promptTokensDetails": [
34+
{
35+
"modality": "TEXT",
36+
"tokenCount": 9,
37+
},
38+
],
3339
"thoughtsTokenCount": 258,
3440
"totalTokenCount": 296,
3541
},
@@ -124,6 +130,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
124130
"raw": {
125131
"candidatesTokenCount": 29,
126132
"promptTokenCount": 9,
133+
"promptTokensDetails": [
134+
{
135+
"modality": "TEXT",
136+
"tokenCount": 9,
137+
},
138+
],
127139
"thoughtsTokenCount": 258,
128140
"totalTokenCount": 296,
129141
},
@@ -162,6 +174,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
162174
"usageMetadata": {
163175
"candidatesTokenCount": 28,
164176
"promptTokenCount": 9,
177+
"promptTokensDetails": [
178+
{
179+
"modality": "TEXT",
180+
"tokenCount": 9,
181+
},
182+
],
165183
"thoughtsTokenCount": 244,
166184
"totalTokenCount": 281,
167185
},
@@ -256,6 +274,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
256274
"raw": {
257275
"candidatesTokenCount": 28,
258276
"promptTokenCount": 9,
277+
"promptTokensDetails": [
278+
{
279+
"modality": "TEXT",
280+
"tokenCount": 9,
281+
},
282+
],
259283
"thoughtsTokenCount": 244,
260284
"totalTokenCount": 281,
261285
},
@@ -280,6 +304,12 @@ exports[`doGenerate > text > should extract usage 1`] = `
280304
"raw": {
281305
"candidatesTokenCount": 28,
282306
"promptTokenCount": 9,
307+
"promptTokensDetails": [
308+
{
309+
"modality": "TEXT",
310+
"tokenCount": 9,
311+
},
312+
],
283313
"thoughtsTokenCount": 244,
284314
"totalTokenCount": 281,
285315
},
@@ -316,6 +346,12 @@ exports[`doGenerate > tool-call > should extract tool calls 1`] = `
316346
"usageMetadata": {
317347
"candidatesTokenCount": 15,
318348
"promptTokenCount": 29,
349+
"promptTokensDetails": [
350+
{
351+
"modality": "TEXT",
352+
"tokenCount": 29,
353+
},
354+
],
319355
"thoughtsTokenCount": 893,
320356
"totalTokenCount": 937,
321357
},
@@ -434,6 +470,12 @@ exports[`doGenerate > tool-call > should extract tool calls 1`] = `
434470
"raw": {
435471
"candidatesTokenCount": 15,
436472
"promptTokenCount": 29,
473+
"promptTokensDetails": [
474+
{
475+
"modality": "TEXT",
476+
"tokenCount": 29,
477+
},
478+
],
437479
"thoughtsTokenCount": 893,
438480
"totalTokenCount": 937,
439481
},
@@ -472,6 +514,12 @@ exports[`doGenerate > tool-call-gemini3 > should extract tool call with thoughtS
472514
"usageMetadata": {
473515
"candidatesTokenCount": 15,
474516
"promptTokenCount": 29,
517+
"promptTokensDetails": [
518+
{
519+
"modality": "TEXT",
520+
"tokenCount": 29,
521+
},
522+
],
475523
"thoughtsTokenCount": 1801,
476524
"totalTokenCount": 1845,
477525
},
@@ -570,6 +618,12 @@ exports[`doGenerate > tool-call-gemini3 > should extract tool call with thoughtS
570618
"raw": {
571619
"candidatesTokenCount": 15,
572620
"promptTokenCount": 29,
621+
"promptTokensDetails": [
622+
{
623+
"modality": "TEXT",
624+
"tokenCount": 29,
625+
},
626+
],
573627
"thoughtsTokenCount": 1801,
574628
"totalTokenCount": 1845,
575629
},
@@ -633,6 +687,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
633687
"usageMetadata": {
634688
"candidatesTokenCount": 29,
635689
"promptTokenCount": 9,
690+
"promptTokensDetails": [
691+
{
692+
"modality": "TEXT",
693+
"tokenCount": 9,
694+
},
695+
],
636696
"thoughtsTokenCount": 256,
637697
"totalTokenCount": 294,
638698
},
@@ -654,6 +714,12 @@ Here is the breakdown: st**r**awbe**rr**y.",
654714
"raw": {
655715
"candidatesTokenCount": 29,
656716
"promptTokenCount": 9,
717+
"promptTokensDetails": [
718+
{
719+
"modality": "TEXT",
720+
"tokenCount": 9,
721+
},
722+
],
657723
"thoughtsTokenCount": 256,
658724
"totalTokenCount": 294,
659725
},
@@ -717,6 +783,12 @@ exports[`doStream > reasoning-gemini3 > should stream reasoning with thoughtSign
717783
"usageMetadata": {
718784
"candidatesTokenCount": 23,
719785
"promptTokenCount": 9,
786+
"promptTokensDetails": [
787+
{
788+
"modality": "TEXT",
789+
"tokenCount": 9,
790+
},
791+
],
720792
"thoughtsTokenCount": 302,
721793
"totalTokenCount": 334,
722794
},
@@ -738,6 +810,12 @@ exports[`doStream > reasoning-gemini3 > should stream reasoning with thoughtSign
738810
"raw": {
739811
"candidatesTokenCount": 23,
740812
"promptTokenCount": 9,
813+
"promptTokensDetails": [
814+
{
815+
"modality": "TEXT",
816+
"tokenCount": 9,
817+
},
818+
],
741819
"thoughtsTokenCount": 302,
742820
"totalTokenCount": 334,
743821
},
@@ -906,6 +984,12 @@ st**r**awbe**rr**y",
906984
"usageMetadata": {
907985
"candidatesTokenCount": 23,
908986
"promptTokenCount": 9,
987+
"promptTokensDetails": [
988+
{
989+
"modality": "TEXT",
990+
"tokenCount": 9,
991+
},
992+
],
909993
"thoughtsTokenCount": 185,
910994
"totalTokenCount": 217,
911995
},
@@ -927,6 +1011,12 @@ st**r**awbe**rr**y",
9271011
"raw": {
9281012
"candidatesTokenCount": 23,
9291013
"promptTokenCount": 9,
1014+
"promptTokensDetails": [
1015+
{
1016+
"modality": "TEXT",
1017+
"tokenCount": 9,
1018+
},
1019+
],
9301020
"thoughtsTokenCount": 185,
9311021
"totalTokenCount": 217,
9321022
},
@@ -997,6 +1087,12 @@ exports[`doStream > tool-call > should stream tool call 1`] = `
9971087
"usageMetadata": {
9981088
"candidatesTokenCount": 15,
9991089
"promptTokenCount": 29,
1090+
"promptTokensDetails": [
1091+
{
1092+
"modality": "TEXT",
1093+
"tokenCount": 29,
1094+
},
1095+
],
10001096
"thoughtsTokenCount": 45,
10011097
"totalTokenCount": 89,
10021098
},
@@ -1018,6 +1114,12 @@ exports[`doStream > tool-call > should stream tool call 1`] = `
10181114
"raw": {
10191115
"candidatesTokenCount": 15,
10201116
"promptTokenCount": 29,
1117+
"promptTokensDetails": [
1118+
{
1119+
"modality": "TEXT",
1120+
"tokenCount": 29,
1121+
},
1122+
],
10211123
"thoughtsTokenCount": 45,
10221124
"totalTokenCount": 89,
10231125
},
@@ -1088,6 +1190,12 @@ exports[`doStream > tool-call-gemini3 > should stream tool call with thoughtSign
10881190
"usageMetadata": {
10891191
"candidatesTokenCount": 15,
10901192
"promptTokenCount": 29,
1193+
"promptTokensDetails": [
1194+
{
1195+
"modality": "TEXT",
1196+
"tokenCount": 29,
1197+
},
1198+
],
10911199
"thoughtsTokenCount": 804,
10921200
"totalTokenCount": 848,
10931201
},
@@ -1109,6 +1217,12 @@ exports[`doStream > tool-call-gemini3 > should stream tool call with thoughtSign
11091217
"raw": {
11101218
"candidatesTokenCount": 15,
11111219
"promptTokenCount": 29,
1220+
"promptTokensDetails": [
1221+
{
1222+
"modality": "TEXT",
1223+
"tokenCount": 29,
1224+
},
1225+
],
11121226
"thoughtsTokenCount": 804,
11131227
"totalTokenCount": 848,
11141228
},

‎packages/google/src/convert-google-generative-ai-usage.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,19 @@
11
import { LanguageModelV3Usage } from '@ai-sdk/provider';
22

3+
export type GoogleGenerativeAITokenDetail = {
4+
modality: string;
5+
tokenCount: number;
6+
};
7+
38
export type GoogleGenerativeAIUsageMetadata = {
49
promptTokenCount?: number | null;
510
candidatesTokenCount?: number | null;
611
totalTokenCount?: number | null;
712
cachedContentTokenCount?: number | null;
813
thoughtsTokenCount?: number | null;
914
trafficType?: string | null;
15+
promptTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
16+
candidatesTokensDetails?: GoogleGenerativeAITokenDetail[] | null;
1017
};
1118

1219
export function convertGoogleGenerativeAIUsage(

‎packages/google/src/google-generative-ai-language-model.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,6 +1110,15 @@ const getSafetyRatingSchema = () =>
11101110
blocked: z.boolean().nullish(),
11111111
});
11121112

1113+
const tokenDetailsSchema = z
1114+
.array(
1115+
z.object({
1116+
modality: z.string(),
1117+
tokenCount: z.number(),
1118+
}),
1119+
)
1120+
.nullish();
1121+
11131122
const usageSchema = z.object({
11141123
cachedContentTokenCount: z.number().nullish(),
11151124
thoughtsTokenCount: z.number().nullish(),
@@ -1118,6 +1127,9 @@ const usageSchema = z.object({
11181127
totalTokenCount: z.number().nullish(),
11191128
// https://cloud.google.com/vertex-ai/generative-ai/docs/reference/rest/v1/GenerateContentResponse#TrafficType
11201129
trafficType: z.string().nullish(),
1130+
// https://ai.google.dev/api/generate-content#Modality
1131+
promptTokensDetails: tokenDetailsSchema,
1132+
candidatesTokensDetails: tokenDetailsSchema,
11211133
});
11221134

11231135
// https://ai.google.dev/api/generate-content#UrlRetrievalMetadata

0 commit comments

Comments
 (0)