Skip to content

Commit fd84be8

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add support for FunctionResponsePart
feat: add support for raw media bytes for function response feat: add Computer Use tool type feat: add support for image generation features feat: add support for additional reasons to stop image generation feat: add URL_RETRIEVAL_STATUS_UNSAFE and URL_RETRIEVAL_STATUS_PAYWALL statuses for url retrieval feat: add log_probability_sum fix!: replaced GenerateVideoResponse by PredictLongRunningGeneratedVideoResponse field PiperOrigin-RevId: 819821787
1 parent 4c5d582 commit fd84be8

File tree

5 files changed

+168
-19
lines changed

5 files changed

+168
-19
lines changed

‎google/ai/generativelanguage/v1beta/content.proto‎

Lines changed: 92 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,29 @@ message Part {
141141
// Optional. An opaque signature for the thought so it can be reused in
142142
// subsequent requests.
143143
bytes thought_signature = 13 [(google.api.field_behavior) = OPTIONAL];
144+
145+
// Custom metadata associated with the Part.
146+
// Agents using genai.Part as content representation may need to keep track
147+
// of the additional information. For example it can be name of a file/source
148+
// from which the Part originates or a way to multiplex multiple Part streams.
149+
google.protobuf.Struct part_metadata = 8;
150+
}
151+
152+
// A datatype containing media that is part of a `FunctionResponse` message.
153+
//
154+
// A `FunctionResponsePart` consists of data which has an associated datatype. A
155+
// `FunctionResponsePart` can only contain one of the accepted types in
156+
// `FunctionResponsePart.data`.
157+
//
158+
// A `FunctionResponsePart` must have a fixed IANA MIME type identifying the
159+
// type and subtype of the media if the `inline_data` field is filled with raw
160+
// bytes.
161+
message FunctionResponsePart {
162+
// The data of the function response part.
163+
oneof data {
164+
// Inline media bytes.
165+
FunctionResponseBlob inline_data = 1;
166+
}
144167
}
145168

146169
// Raw media bytes.
@@ -160,6 +183,24 @@ message Blob {
160183
bytes data = 2;
161184
}
162185

186+
// Raw media bytes for function response.
187+
//
188+
// Text should not be sent as raw bytes, use the 'FunctionResponse.response'
189+
// field.
190+
message FunctionResponseBlob {
191+
// The IANA standard MIME type of the source data.
192+
// Examples:
193+
// - image/png
194+
// - image/jpeg
195+
// If an unsupported MIME type is provided, an error will be returned. For a
196+
// complete list of supported types, see [Supported file
197+
// formats](https://ai.google.dev/gemini-api/docs/prompting_with_media#supported_file_formats).
198+
string mime_type = 1;
199+
200+
// Raw bytes for media formats.
201+
bytes data = 2;
202+
}
203+
163204
// URI based data.
164205
message FileData {
165206
// Optional. The IANA standard MIME type of the source data.
@@ -242,6 +283,8 @@ message CodeExecutionResult {
242283
// A `Tool` is a piece of code that enables the system to interact with
243284
// external systems to perform an action, or set of actions, outside of
244285
// knowledge and scope of the model.
286+
//
287+
// Next ID: 12
245288
message Tool {
246289
// GoogleSearch tool type.
247290
// Tool to support Google Search in Model. Powered by Google.
@@ -253,6 +296,29 @@ message Tool {
253296
[(google.api.field_behavior) = OPTIONAL];
254297
}
255298

299+
// Computer Use tool type.
300+
message ComputerUse {
301+
// Represents the environment being operated, such as a web browser.
302+
enum Environment {
303+
// Defaults to browser.
304+
ENVIRONMENT_UNSPECIFIED = 0;
305+
306+
// Operates in a web browser.
307+
ENVIRONMENT_BROWSER = 1;
308+
}
309+
310+
// Required. The environment being operated.
311+
Environment environment = 3 [(google.api.field_behavior) = REQUIRED];
312+
313+
// Optional. By default, predefined functions are included in the final
314+
// model call. Some of them can be explicitly excluded from being
315+
// automatically included. This can serve two purposes:
316+
// 1. Using a more restricted / different action space.
317+
// 2. Improving the definitions / instructions of predefined functions.
318+
repeated string excluded_predefined_functions = 5
319+
[(google.api.field_behavior) = OPTIONAL];
320+
}
321+
256322
// Optional. A list of `FunctionDeclarations` available to the model that can
257323
// be used for function calling.
258324
//
@@ -280,6 +346,11 @@ message Tool {
280346
// Tool to support Google Search in Model. Powered by Google.
281347
GoogleSearch google_search = 4 [(google.api.field_behavior) = OPTIONAL];
282348

349+
// Optional. Tool to support the model interacting directly with the computer.
350+
// If enabled, it automatically populates computer-use specific Function
351+
// Declarations.
352+
ComputerUse computer_use = 6 [(google.api.field_behavior) = OPTIONAL];
353+
283354
// Optional. Tool to support URL context retrieval.
284355
UrlContext url_context = 8 [(google.api.field_behavior) = OPTIONAL];
285356
}
@@ -352,6 +423,9 @@ message FunctionCallingConfig {
352423
// Model decides to predict either a function call
353424
// or a natural language response, but will validate function calls with
354425
// constrained decoding.
426+
// If "allowed_function_names" are set, the predicted function call will be
427+
// limited to any one of "allowed_function_names", else the predicted
428+
// function call will be any one of the provided "function_declarations".
355429
VALIDATED = 4;
356430
}
357431

@@ -362,9 +436,9 @@ message FunctionCallingConfig {
362436
// Optional. A set of function names that, when provided, limits the functions
363437
// the model will call.
364438
//
365-
// This should only be set when the Mode is ANY. Function names
366-
// should match [FunctionDeclaration.name]. With mode set to ANY, model will
367-
// predict a function call from the set of function names provided.
439+
// This should only be set when the Mode is ANY or VALIDATED. Function names
440+
// should match [FunctionDeclaration.name]. When set, model will
441+
// predict a function call from only allowed function names.
368442
repeated string allowed_function_names = 2
369443
[(google.api.field_behavior) = OPTIONAL];
370444
}
@@ -392,8 +466,8 @@ message FunctionDeclaration {
392466
}
393467

394468
// Required. The name of the function.
395-
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
396-
// length of 63.
469+
// Must be a-z, A-Z, 0-9, or contain underscores, colons, dots, and dashes,
470+
// with a maximum length of 64.
397471
string name = 1 [(google.api.field_behavior) = REQUIRED];
398472

399473
// Required. A brief description of the function.
@@ -453,7 +527,7 @@ message FunctionCall {
453527

454528
// Required. The name of the function to call.
455529
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
456-
// length of 63.
530+
// length of 64.
457531
string name = 1 [(google.api.field_behavior) = REQUIRED];
458532

459533
// Optional. The function parameters and values in JSON object format.
@@ -491,12 +565,21 @@ message FunctionResponse {
491565

492566
// Required. The name of the function to call.
493567
// Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum
494-
// length of 63.
568+
// length of 64.
495569
string name = 1 [(google.api.field_behavior) = REQUIRED];
496570

497571
// Required. The function response in JSON object format.
572+
// Callers can use any keys of their choice that fit the function's syntax
573+
// to return the function output, e.g. "output", "result", etc.
574+
// In particular, if the function call failed to execute, the response can
575+
// have an "error" key to return error details to the model.
498576
google.protobuf.Struct response = 2 [(google.api.field_behavior) = REQUIRED];
499577

578+
// Optional. Ordered `Parts` that constitute a function response. Parts may
579+
// have different IANA MIME types.
580+
repeated FunctionResponsePart parts = 8
581+
[(google.api.field_behavior) = OPTIONAL];
582+
500583
// Optional. Signals that function call continues, and more responses will be
501584
// returned, turning the function call into a generator.
502585
// Is only applicable to NON_BLOCKING function calls, is ignored otherwise.
@@ -521,11 +604,8 @@ message Schema {
521604
// Required. Data type.
522605
Type type = 1 [(google.api.field_behavior) = REQUIRED];
523606

524-
// Optional. The format of the data. This is used only for primitive
525-
// datatypes. Supported formats:
526-
// for NUMBER type: float, double
527-
// for INTEGER type: int32, int64
528-
// for STRING type: enum, date-time
607+
// Optional. The format of the data. Any value is allowed, but most do not
608+
// trigger any special functionality.
529609
string format = 2 [(google.api.field_behavior) = OPTIONAL];
530610

531611
// Optional. The title of the schema.

‎google/ai/generativelanguage/v1beta/file.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ message File {
6161

6262
// Indicates the file is generated by Google.
6363
GENERATED = 2;
64+
65+
// Indicates the file is a registered, i.e. a Google Cloud Storage file.
66+
REGISTERED = 3;
6467
}
6568

6669
// Metadata for the File.

‎google/ai/generativelanguage/v1beta/generative_service.proto‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ enum TaskType {
163163
}
164164

165165
// Request to generate a completion from the model.
166+
// NEXT ID: 18
166167
message GenerateContentRequest {
167168
// Required. The name of the `Model` to use for generating the completion.
168169
//
@@ -303,8 +304,19 @@ message ThinkingConfig {
303304
optional int32 thinking_budget = 2;
304305
}
305306

307+
// Config for image generation features.
308+
message ImageConfig {
309+
// Optional. The aspect ratio of the image to generate. Supported aspect
310+
// ratios: 1:1, 2:3, 3:2, 3:4, 4:3, 9:16, 16:9, 21:9.
311+
//
312+
// If not specified, the model will choose a default aspect ratio based on any
313+
// reference images provided.
314+
optional string aspect_ratio = 1 [(google.api.field_behavior) = OPTIONAL];
315+
}
316+
306317
// Configuration options for model generation and outputs. Not all parameters
307318
// are configurable for every model.
319+
// Next ID: 29
308320
message GenerationConfig {
309321
// Supported modalities of the response.
310322
enum Modality {
@@ -450,8 +462,17 @@ message GenerationConfig {
450462
// be used within non-required properties. (Nullable properties are not
451463
// sufficient.) If `$ref` is set on a sub-schema, no other properties, except
452464
// for than those starting as a `$`, may be set.
453-
google.protobuf.Value response_json_schema = 24
454-
[(google.api.field_behavior) = OPTIONAL];
465+
google.protobuf.Value response_json_schema = 24 [
466+
json_name = "_responseJsonSchema",
467+
(google.api.field_behavior) = OPTIONAL
468+
];
469+
470+
// Optional. An internal detail. Use `responseJsonSchema` rather than this
471+
// field.
472+
google.protobuf.Value response_json_schema_ordered = 28 [
473+
json_name = "responseJsonSchema",
474+
(google.api.field_behavior) = OPTIONAL
475+
];
455476

456477
// Optional. Presence penalty applied to the next token's logprobs if the
457478
// token has already been seen in the response.
@@ -494,6 +515,7 @@ message GenerationConfig {
494515
// [response_logprobs=True][google.ai.generativelanguage.v1beta.GenerationConfig.response_logprobs].
495516
// This sets the number of top logprobs to return at each decoding step in the
496517
// [Candidate.logprobs_result][google.ai.generativelanguage.v1beta.Candidate.logprobs_result].
518+
// The number must be in the range of [0, 20].
497519
optional int32 logprobs = 18 [(google.api.field_behavior) = OPTIONAL];
498520

499521
// Optional. Enables enhanced civic answers. It may not be available for all
@@ -523,6 +545,12 @@ message GenerationConfig {
523545
optional ThinkingConfig thinking_config = 22
524546
[(google.api.field_behavior) = OPTIONAL];
525547

548+
// Optional. Config for image generation.
549+
// An error will be returned if this field is set for models that don't
550+
// support these config options.
551+
optional ImageConfig image_config = 27
552+
[(google.api.field_behavior) = OPTIONAL];
553+
526554
// Optional. If specified, the media resolution specified will be used.
527555
optional MediaResolution media_resolution = 23
528556
[(google.api.field_behavior) = OPTIONAL];
@@ -700,8 +728,25 @@ message Candidate {
700728
// violations.
701729
IMAGE_SAFETY = 11;
702730

731+
// Image generation stopped because generated images has other prohibited
732+
// content.
733+
IMAGE_PROHIBITED_CONTENT = 14;
734+
735+
// Image generation stopped because of other miscellaneous issue.
736+
IMAGE_OTHER = 15;
737+
738+
// The model was expected to generate an image, but none was generated.
739+
NO_IMAGE = 16;
740+
741+
// Image generation stopped due to recitation.
742+
IMAGE_RECITATION = 17;
743+
703744
// Model generated a tool call but no tools were enabled in the request.
704745
UNEXPECTED_TOOL_CALL = 12;
746+
747+
// Model called too many tools consecutively, thus the system exited
748+
// execution.
749+
TOO_MANY_TOOL_CALLS = 13;
705750
}
706751

707752
// Output only. Index of the candidate in the list of response candidates.
@@ -718,6 +763,13 @@ message Candidate {
718763
(google.api.field_behavior) = OUTPUT_ONLY
719764
];
720765

766+
// Optional. Output only. Details the reason why the model stopped generating
767+
// tokens. This is populated only when `finish_reason` is set.
768+
optional string finish_message = 4 [
769+
(google.api.field_behavior) = OPTIONAL,
770+
(google.api.field_behavior) = OUTPUT_ONLY
771+
];
772+
721773
// List of ratings for the safety of a response candidate.
722774
//
723775
// There is at most one rating per category.
@@ -777,6 +829,12 @@ message UrlMetadata {
777829

778830
// Url retrieval is failed due to error.
779831
URL_RETRIEVAL_STATUS_ERROR = 2;
832+
833+
// Url retrieval is failed because the content is behind paywall.
834+
URL_RETRIEVAL_STATUS_PAYWALL = 3;
835+
836+
// Url retrieval is failed because the content is unsafe.
837+
URL_RETRIEVAL_STATUS_UNSAFE = 4;
780838
}
781839

782840
// Retrieved url by the tool.
@@ -806,6 +864,9 @@ message LogprobsResult {
806864
repeated Candidate candidates = 1;
807865
}
808866

867+
// Sum of log probabilities for all tokens.
868+
optional float log_probability_sum = 3;
869+
809870
// Length = total number of decoding steps.
810871
repeated TopCandidates top_candidates = 1;
811872

@@ -1552,8 +1613,7 @@ message BidiGenerateContentToolResponse {
15521613
message BidiGenerateContentClientMessage {
15531614
// The type of the message.
15541615
oneof message_type {
1555-
// Optional. Session configuration sent in the first and only first client
1556-
// message.
1616+
// Optional. Session configuration sent only in the first client message.
15571617
BidiGenerateContentSetup setup = 1 [(google.api.field_behavior) = OPTIONAL];
15581618

15591619
// Optional. Incremental update of the current conversation delivered from
@@ -1625,6 +1685,11 @@ message BidiGenerateContentServerContent {
16251685

16261686
UrlContextMetadata url_context_metadata = 9
16271687
[(google.api.field_behavior) = OUTPUT_ONLY];
1688+
1689+
// Output only. If true, indicates that the model is not generating content
1690+
// because it is waiting for more input from the user, e.g. because it expects
1691+
// the user to continue talking.
1692+
bool waiting_for_input = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
16281693
}
16291694

16301695
// Request for the client to execute the `function_calls` and return the

‎google/ai/generativelanguage/v1beta/prediction_service.proto‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ message PredictLongRunningResponse {
106106
// The response of the long running operation.
107107
oneof response {
108108
// The response of the video generation prediction.
109-
GenerateVideoResponse generate_video_response = 1;
109+
PredictLongRunningGeneratedVideoResponse generate_video_response = 1;
110110
}
111111
}
112112

@@ -135,7 +135,7 @@ message Video {
135135
}
136136

137137
// Veo response.
138-
message GenerateVideoResponse {
138+
message PredictLongRunningGeneratedVideoResponse {
139139
// The generated samples.
140140
repeated Media generated_samples = 1;
141141

‎google/ai/generativelanguage/v1beta/safety.proto‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ enum HarmCategory {
6565
HARM_CATEGORY_DANGEROUS_CONTENT = 10;
6666

6767
// **Gemini** - Content that may be used to harm civic integrity.
68-
HARM_CATEGORY_CIVIC_INTEGRITY = 11;
68+
// DEPRECATED: use enable_enhanced_civic_answers instead.
69+
HARM_CATEGORY_CIVIC_INTEGRITY = 11 [deprecated = true];
6970
}
7071

7172
// Content filtering metadata associated with processing a single request.

0 commit comments

Comments
 (0)