feat: [generativelanguage] add support for GoogleMaps and FileSearch tools (#6957)

* feat: add IMAGE_RECITATION reason when generation is stopped due to recitation
feat: add support for GroundingMetadata.google_maps_widget_context_token
feat: add json_name for GenerationConfig.response_json_schema_ordered field

PiperOrigin-RevId: 846116816

Source-Link: googleapis/googleapis@c631c69

Source-Link: googleapis/googleapis-gen@287df78
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiMjg3ZGY3ODg5NWI0N2YxMjlhN2Y4OGYwZTlhYTFhOTYyN2ZmMTVkZSJ9

* feat: add support for GoogleMaps and FileSearch tools
feat: add support for RetrievalConfig

PiperOrigin-RevId: 846118142

Source-Link: googleapis/googleapis@bc7e3ba

Source-Link: googleapis/googleapis-gen@f9c0ecd
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWFpLWdlbmVyYXRpdmVsYW5ndWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiZjljMGVjZGVhN2FlMzk0OGVkZDdlZWI1NTQ5NmY3ODUyNzljNmZlNSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: gcf-owl-bot[bot]

packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1/generative_service.proto

@@ -145,7 +145,6 @@ enum TaskType {
 }
 
 // Request to generate a completion from the model.
-// NEXT ID: 18
 message GenerateContentRequest {
   // Required. The name of the `Model` to use for generating the completion.
   //
@@ -192,7 +191,6 @@ message GenerateContentRequest {
 
 // Configuration options for model generation and outputs. Not all parameters
 // are configurable for every model.
-// Next ID: 29
 message GenerationConfig {
   // Optional. Number of generated responses to return. If unset, this will
   // default to 1. Please note that this doesn't work for previous generation
@@ -253,8 +251,10 @@ message GenerationConfig {
 
   // Optional. An internal detail. Use `responseJsonSchema` rather than this
   // field.
-  google.protobuf.Value response_json_schema_ordered = 28
-      [(google.api.field_behavior) = OPTIONAL];
+  google.protobuf.Value response_json_schema_ordered = 28 [
+    json_name = "responseJsonSchema",
+    (google.api.field_behavior) = OPTIONAL
+  ];
 
   // Optional. Presence penalty applied to the next token's logprobs if the
   // token has already been seen in the response.
@@ -461,6 +461,9 @@ message Candidate {
     // The model was expected to generate an image, but none was generated.
     NO_IMAGE = 16;
 
+    // Image generation stopped due to recitation.
+    IMAGE_RECITATION = 17;
+
     // Model generated a tool call but no tools were enabled in the request.
     UNEXPECTED_TOOL_CALL = 12;
 
@@ -616,6 +619,13 @@ message GroundingMetadata {
 
   // Web search queries for the following-up web search.
   repeated string web_search_queries = 5;
+
+  // Optional. Resource name of the Google Maps widget context token that can be
+  // used with the PlacesContextElement widget in order to render contextual
+  // data. Only populated in the case that grounding with Google Maps is
+  // enabled.
+  optional string google_maps_widget_context_token = 7
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Google search entry point.

packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/content.proto

@@ -20,6 +20,7 @@ import "google/api/field_behavior.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/type/interval.proto";
+import "google/type/latlng.proto";
 
 option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1beta/generativelanguagepb;generativelanguagepb";
 option java_multiple_files = true;
@@ -353,11 +354,58 @@ message Tool {
 
   // Optional. Tool to support URL context retrieval.
   UrlContext url_context = 8 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. FileSearch tool type.
+  // Tool to retrieve knowledge from Semantic Retrieval corpora.
+  FileSearch file_search = 9 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Tool that allows grounding the model's response with geospatial
+  // context related to the user's query.
+  GoogleMaps google_maps = 11 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// The GoogleMaps Tool that provides geospatial context for the user's query.
+message GoogleMaps {
+  // Optional. Whether to return a widget context token in the GroundingMetadata
+  // of the response. Developers can use the widget context token to render a
+  // Google Maps widget with geospatial context related to the places that the
+  // model references in the response.
+  bool enable_widget = 1 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Tool to support URL context retrieval.
 message UrlContext {}
 
+// The FileSearch tool that retrieves knowledge from Semantic Retrieval corpora.
+// Files are imported to Semantic Retrieval corpora using the ImportFile API.
+message FileSearch {
+  // The semantic retrieval resource to retrieve from.
+  message RetrievalResource {
+    // Required. The name of the semantic retrieval resource to retrieve from.
+    // Example: `ragStores/my-rag-store-123`
+    string rag_store_name = 1 [(google.api.field_behavior) = REQUIRED];
+  }
+
+  // Semantic retrieval configuration.
+  message RetrievalConfig {
+    // Optional. The number of semantic retrieval chunks to retrieve.
+    optional int32 top_k = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Metadata filter to apply to the semantic retrieval documents
+    // and chunks.
+    string metadata_filter = 3 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Required. Semantic retrieval resources to retrieve from.
+  // Currently only supports one corpus. In the future we may open up multiple
+  // corpora support.
+  repeated RetrievalResource retrieval_resources = 1
+      [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. The configuration for the retrieval.
+  RetrievalConfig retrieval_config = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
 // Tool to retrieve public web data for grounding, powered by Google.
 message GoogleSearchRetrieval {
   // Specifies the dynamic retrieval configuration for the given source.
@@ -396,6 +444,20 @@ message ToolConfig {
   // Optional. Function calling config.
   FunctionCallingConfig function_calling_config = 1
       [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Retrieval config.
+  RetrievalConfig retrieval_config = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Retrieval config.
+message RetrievalConfig {
+  // Optional. The location of the user.
+  google.type.LatLng lat_lng = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. The language code of the user.
+  // Language code for content. Use language tags defined by
+  // [BCP47](https://www.rfc-editor.org/rfc/bcp/bcp47.txt).
+  string language_code = 2 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Configuration for specifying function calling behavior.

packages/google-ai-generativelanguage/protos/google/ai/generativelanguage/v1beta/generative_service.proto

@@ -163,7 +163,6 @@ enum TaskType {
 }
 
 // Request to generate a completion from the model.
-// NEXT ID: 18
 message GenerateContentRequest {
   // Required. The name of the `Model` to use for generating the completion.
   //
@@ -316,7 +315,6 @@ message ImageConfig {
 
 // Configuration options for model generation and outputs. Not all parameters
 // are configurable for every model.
-// Next ID: 29
 message GenerationConfig {
   // Supported modalities of the response.
   enum Modality {
@@ -947,6 +945,13 @@ message GroundingMetadata {
 
   // Web search queries for the following-up web search.
   repeated string web_search_queries = 5;
+
+  // Optional. Resource name of the Google Maps widget context token that can be
+  // used with the PlacesContextElement widget in order to render contextual
+  // data. Only populated in the case that grounding with Google Maps is
+  // enabled.
+  optional string google_maps_widget_context_token = 7
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Google search entry point.
@@ -971,10 +976,75 @@ message GroundingChunk {
     optional string title = 2;
   }
 
+  // Chunk from context retrieved by the file search tool.
+  message RetrievedContext {
+    // Optional. URI reference of the semantic retrieval document.
+    optional string uri = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Title of the document.
+    optional string title = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Text of the chunk.
+    optional string text = 3 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // A grounding chunk from Google Maps. A Maps chunk corresponds to a single
+  // place.
+  message Maps {
+    // Collection of sources that provide answers about the features of a given
+    // place in Google Maps. Each PlaceAnswerSources message corresponds to a
+    // specific place in Google Maps. The Google Maps tool used these sources in
+    // order to answer questions about features of the place (e.g: "does Bar Foo
+    // have Wifi" or "is Foo Bar wheelchair accessible?"). Currently we only
+    // support review snippets as sources.
+    message PlaceAnswerSources {
+      // Encapsulates a snippet of a user review that answers a question about
+      // the features of a specific place in Google Maps.
+      message ReviewSnippet {
+        // The ID of the review snippet.
+        optional string review_id = 1;
+
+        // A link that corresponds to the user review on Google Maps.
+        optional string google_maps_uri = 2;
+
+        // Title of the review.
+        optional string title = 3;
+      }
+
+      // Snippets of reviews that are used to generate answers about the
+      // features of a given place in Google Maps.
+      repeated ReviewSnippet review_snippets = 1;
+    }
+
+    // URI reference of the place.
+    optional string uri = 1;
+
+    // Title of the place.
+    optional string title = 2;
+
+    // Text description of the place answer.
+    optional string text = 3;
+
+    // This ID of the place, in `places/{place_id}` format. A user can use this
+    // ID to look up that place.
+    optional string place_id = 4;
+
+    // Sources that provide answers about the features of a given place in
+    // Google Maps.
+    optional PlaceAnswerSources place_answer_sources = 5;
+  }
+
   // Chunk type.
   oneof chunk_type {
     // Grounding chunk from the web.
     Web web = 1;
+
+    // Optional. Grounding chunk from context retrieved by the file search tool.
+    RetrievedContext retrieved_context = 2
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Grounding chunk from Google Maps.
+    Maps maps = 3 [(google.api.field_behavior) = OPTIONAL];
   }
 }