feat(sources/elasticsearch): add Elasticsearch source and tools (googleapis#1109)

Add support for Elasticsearch with the following tools:
* search
* esql
* get_mappings
* list_indices

This PR fixes googleapis#859

---------

Co-authored-by: duwenxin <[email protected]>
Co-authored-by: Wenxin Du <[email protected]>

Author: Anaethelion

.ci/integration.cloudbuild.yaml

@@ -705,7 +705,26 @@ steps:
       - |
         ./yugabytedb.test -test.v
 
- 
+  - id: "elasticsearch"
+    name: golang:1
+    waitFor: ["compile-test-binary"]
+    entrypoint: /bin/bash
+    env:
+      - "GOPATH=/gopath"
+      - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL"
+    secretEnv: ["CLIENT_ID", "ELASTICSEARCH_USER", "ELASTICSEARCH_PASS", "ELASTICSEARCH_HOST"]
+    volumes:
+      - name: "go"
+        path: "/gopath"
+    args:
+      - -c
+      - |
+        .ci/test_with_coverage.sh \
+          "Elasticsearch" \
+          elasticsearch \
+          elasticsearch
+
+
   - id: "cassandra"
     name: golang:1
     waitFor: ["compile-test-binary"]
@@ -764,7 +783,7 @@ steps:
         .ci/test_with_coverage.sh \
           "Serverless Spark" \
           serverlessspark
-            
+
 availableSecrets:
   secretManager:
     - versionName: projects/$PROJECT_ID/secrets/cloud_sql_pg_user/versions/latest
@@ -855,6 +874,12 @@ availableSecrets:
       env: YUGABYTEDB_USER
     - versionName: projects/$PROJECT_ID/secrets/yugabytedb_pass/versions/latest
       env: YUGABYTEDB_PASS
+    - versionName: projects/$PROJECT_ID/secrets/elastic_search_host/versions/latest
+      env: ELASTICSEARCH_HOST
+    - versionName: projects/$PROJECT_ID/secrets/elastic_search_user/versions/latest
+      env: ELASTICSEARCH_USER
+    - versionName: projects/$PROJECT_ID/secrets/elastic_search_pass/versions/latest
+      env: ELASTICSEARCH_PASS
     - versionName: projects/$PROJECT_ID/secrets/cassandra_user/versions/latest
       env: CASSANDRA_USER
     - versionName: projects/$PROJECT_ID/secrets/cassandra_pass/versions/latest

cmd/root.go

@@ -85,6 +85,7 @@ import (
 	_ "github.com/googleapis/genai-toolbox/internal/tools/dataplex/dataplexsearchaspecttypes"
 	_ "github.com/googleapis/genai-toolbox/internal/tools/dataplex/dataplexsearchentries"
 	_ "github.com/googleapis/genai-toolbox/internal/tools/dgraph"
+	_ "github.com/googleapis/genai-toolbox/internal/tools/elasticsearch/elasticsearchesql"
 	_ "github.com/googleapis/genai-toolbox/internal/tools/firebird/firebirdexecutesql"
 	_ "github.com/googleapis/genai-toolbox/internal/tools/firebird/firebirdsql"
 	_ "github.com/googleapis/genai-toolbox/internal/tools/firestore/firestoreadddocuments"
@@ -195,6 +196,7 @@ import (
 	_ "github.com/googleapis/genai-toolbox/internal/sources/couchbase"
 	_ "github.com/googleapis/genai-toolbox/internal/sources/dataplex"
 	_ "github.com/googleapis/genai-toolbox/internal/sources/dgraph"
+	_ "github.com/googleapis/genai-toolbox/internal/sources/elasticsearch"
 	_ "github.com/googleapis/genai-toolbox/internal/sources/firebird"
 	_ "github.com/googleapis/genai-toolbox/internal/sources/firestore"
 	_ "github.com/googleapis/genai-toolbox/internal/sources/http"

docs/en/resources/sources/elasticsearch.md

@@ -0,0 +1,68 @@
+---
+title: "Elasticsearch"
+type: docs
+weight: 1
+description: >
+  Elasticsearch is a distributed, free and open search and analytics engine 
+  for all types of data, including textual, numerical, geospatial, structured, 
+  and unstructured.
+---
+
+# Elasticsearch Source
+
+[Elasticsearch][elasticsearch-docs] is a distributed, free and open search and analytics engine
+for all types of data, including textual, numerical, geospatial, structured,
+and unstructured.
+
+If you are new to Elasticsearch, you can learn how to 
+[set up a cluster and start indexing data][elasticsearch-quickstart].
+
+Elasticsearch uses [ES|QL][elasticsearch-esql] for querying data. ES|QL
+is a powerful query language that allows you to search and aggregate data in
+Elasticsearch.
+
+See the [official documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) for more information.
+
+[elasticsearch-docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
+[elasticsearch-quickstart]: https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started.html
+[elasticsearch-esql]: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
+
+## Available Tools
+
+- [`elasticsearch-esql`](../tools/elasticsearch/elasticsearch-esql.md)
+  Execute ES|QL queries.
+
+## Requirements
+
+### API Key
+
+Toolbox uses an [API key][api-key] to authorize and authenticate when
+interacting with [Elasticsearch][elasticsearch-docs].
+
+In addition to [setting the API key for your server][set-api-key], you need to
+ensure the API key has the correct permissions for the queries you intend to
+run. See [API key management][api-key-management] for more information on
+applying permissions to an API key.
+
+[api-key]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html
+[set-api-key]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html
+[api-key-management]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html
+
+## Example
+
+```yaml
+sources:
+  my-elasticsearch-source:
+    kind: "elasticsearch"
+    addresses:
+      - "http://localhost:9200"
+    apikey: "my-api-key"
+```
+
+## Reference
+
+| **field** | **type** | **required** | **description**                                                               |
+|-----------|:--------:|:------------:|-------------------------------------------------------------------------------|
+| kind      |  string  |     true     | Must be "elasticsearch".                                                      |
+| addresses | []string |     true     | List of Elasticsearch hosts to connect to.                                    |
+| apikey    |  string  |     true     | The API key to use for authentication.                                        |

docs/en/resources/tools/elasticsearch/_index.md

@@ -0,0 +1,7 @@
+---
+title: "Elasticsearch"
+type: docs
+weight: 1
+description: > 
+  Tools that work with Elasticsearch Sources.
+---

docs/en/resources/tools/elasticsearch/elasticsearch-esql.md

@@ -0,0 +1,45 @@
+---
+title: "elasticsearch-esql"
+type: docs
+weight: 2
+description: >
+  Execute ES|QL queries.
+---
+
+# elasticsearch-esql
+
+Execute ES|QL queries.
+
+This tool allows you to execute ES|QL queries against your Elasticsearch
+cluster. You can use this to perform complex searches and aggregations.
+
+See the [official documentation](https://www.elastic.co/docs/reference/query-languages/esql/esql-getting-started) for more information.
+
+## Example
+
+```yaml
+tools:
+  query_my_index:
+    kind: elasticsearch-esql
+    source: elasticsearch-source
+    description: Use this tool to execute ES|QL queries.
+    query: |
+      FROM my-index
+      | KEEP *
+      | LIMIT ?limit
+    parameters:
+      - name: limit
+        type: integer
+        description: Limit the number of results.
+        required: true
+```
+
+## Parameters
+
+| **name**   | **type** | **required** | **description**                                                                                                                                     |
+|------------|:--------:|:------------:|-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| query      |  string  |    false     | The ES\|QL query to run. Can also be passed by parameters.                                                                                          |
+| format     | string |    false     | The format of the query. Default is json. Valid values are csv, json, tsv, txt, yaml, cbor, smile, or arrow.                                        |
+| timeout    | integer |    false     | The timeout for the query in seconds. Default is 60 (1 minute).                                                                                     |
+| parameters | [parameters](../#specifying-parameters) |    false     | List of [parameters](../#specifying-parameters) that will be used with the ES\|QL query.<br/>Only supports “string”, “integer”, “float”, “boolean”. |
+

go.mod

@@ -21,6 +21,8 @@ require (
 	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/couchbase/gocb/v2 v2.11.1
 	github.com/couchbase/tools-common/http v1.0.9
+	github.com/elastic/elastic-transport-go/v8 v8.7.0
+	github.com/elastic/go-elasticsearch/v8 v8.19.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-chi/chi/v5 v5.2.3
 	github.com/go-chi/httplog/v2 v2.1.1

go.sum

@@ -820,6 +820,10 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE=
+github.com/elastic/elastic-transport-go/v8 v8.7.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
+github.com/elastic/go-elasticsearch/v8 v8.19.0 h1:VmfBLNRORY7RZL+9hTxBD97ehl9H8Nxf2QigDh6HuMU=
+github.com/elastic/go-elasticsearch/v8 v8.19.0/go.mod h1:F3j9e+BubmKvzvLjNui/1++nJuJxbkhHefbaT0kFKGY=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=

internal/prebuiltconfigs/prebuiltconfigs_test.go

@@ -36,6 +36,7 @@ var expectedToolSources = []string{
 	"cloud-sql-postgres-observability",
 	"cloud-sql-postgres",
 	"dataplex",
+	"elasticsearch",
 	"firestore",
 	"looker-conversational-analytics",
 	"looker",

internal/prebuiltconfigs/tools/elasticsearch.yaml

@@ -0,0 +1,33 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sources:
+  elasticsearch-source:
+    kind: elasticsearch
+    addresses:
+      - ${ELASTICSEARCH_HOST}
+    apikey: ${ELASTICSEARCH_APIKEY}
+
+tools:
+  execute_esql_query:
+    kind: elasticsearch-esql
+    source: elasticsearch-source
+    description: Use this tool to execute ES|QL queries.
+    parameters:
+      - name: query
+        type: string
+        description: The ES|QL query to execute.
+toolsets:
+  elasticsearch-tools:
+    - execute_esql_query