elastic · ycombinator · Jul 14, 2018 · May 31, 2018 · Jul 11, 2018 · Jul 11, 2018
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -303,6 +303,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Added Traefik module with health metricset. {pull}7413[7413]
 - Add Elasticsearch ml_job metricsets. {pull}7196[7196]
 - Add support for bearer token files to HTTP helper. {pull}7527[7527]
+- Add Elasticsearch index recovery metricset. {pull}7225[7225]
 
 *Packetbeat*
 

@@ -33,6 +33,9 @@ def compose_up(cls):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
             return
 
+        if os.environ.get('NO_COMPOSE'):
+            return
+
         def print_logs(container):
             print("---- " + container.name_without_project)
             print(container.logs())

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3521,6 +3521,113 @@ format: bytes
 Total number of memory used by the segments in bytes.
 
 
+--
+
+[float]
+== index.recovery fields
+
+index
+
+
+
+*`elasticsearch.index.recovery.id`*::
++
+--
+type: long
+
+Shard recovery id.
+
+
+--
+
+*`elasticsearch.index.recovery.type`*::
++
+--
+type: keyword
+
+Shard recovery type.
+
+
+--
+
+*`elasticsearch.index.recovery.primary`*::
++
+--
+type: boolean
+
+True if primary shard.
+
+
+--
+
+*`elasticsearch.index.recovery.stage`*::
++
+--
+type: keyword
+
+Recovery stage.
+
+
+--
+
+*`elasticsearch.index.recovery.target.id`*::
++
+--
+type: keyword
+
+Target node id.
+
+
+--
+
+*`elasticsearch.index.recovery.target.host`*::
++
+--
+type: ip
+
+Target node host address.
+
+
+--
+
+*`elasticsearch.index.recovery.target.name`*::
++
+--
+type: keyword
+
+Target node name.
+
+
+--
+
+*`elasticsearch.index.recovery.source.id`*::
++
+--
+type: keyword
+
+Source node id.
+
+
+--
+
+*`elasticsearch.index.recovery.source.host`*::
++
+--
+type: ip
+
+Source node host address.
+
+
+--
+
+*`elasticsearch.index.recovery.source.name`*::
++
+--
+type: keyword
+
+Source node name.
+
+
 --
 
 [float]

diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc
@@ -26,11 +26,15 @@ metricbeat.modules:
     - node
     - node_stats
     #- index
+    #- index_recovery
     #- index_summary
     #- shard
     #- ml_job
   period: 10s
   hosts: ["localhost:9200"]
+
+  # Set to false to fetch all entries
+  #index_recovery.active_only: true
 ----
 
 This module supports TLS connection when using `ssl` config field, as described in <<configuration-ssl>>. It also supports the options described in <<module-http-config-options>>.
@@ -42,6 +46,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-elasticsearch-index,index>>
 
+* <<metricbeat-metricset-elasticsearch-index_recovery,index_recovery>>
+
 * <<metricbeat-metricset-elasticsearch-index_summary,index_summary>>
 
 * <<metricbeat-metricset-elasticsearch-ml_job,ml_job>>
@@ -56,6 +62,8 @@ The following metricsets are available:
 
 include::elasticsearch/index.asciidoc[]
 
+include::elasticsearch/index_recovery.asciidoc[]
+
 include::elasticsearch/index_summary.asciidoc[]
 
 include::elasticsearch/ml_job.asciidoc[]

diff --git a/metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc b/metricbeat/docs/modules/elasticsearch/index_recovery.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-index_recovery]]
+=== Elasticsearch index_recovery metricset
+
+experimental[]
+
+include::../../../module/elasticsearch/index_recovery/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/index_recovery/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -33,7 +33,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.7+| .7+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+.8+| .8+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+|<<metricbeat-metricset-elasticsearch-index_recovery,index_recovery>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-index_summary,index_summary>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-ml_job,ml_job>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-node,node>> beta[]  

diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -199,12 +199,16 @@ metricbeat.modules:
     - node
     - node_stats
     #- index
+    #- index_recovery
     #- index_summary
     #- shard
     #- ml_job
   period: 10s
   hosts: ["localhost:9200"]
 
+  # Set to false to fetch all entries
+  #index_recovery.active_only: true
+
 #-------------------------------- Etcd Module --------------------------------
 - module: etcd
   metricsets: ["leader", "self", "store"]

diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml
@@ -3,8 +3,12 @@
     - node
     - node_stats
     #- index
+    #- index_recovery
     #- index_summary
     #- shard
     #- ml_job
   period: 10s
   hosts: ["localhost:9200"]
+
+  # Set to false to fetch all entries
+  #index_recovery.active_only: true
diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go
@@ -118,15 +118,8 @@ func getMasterName(http *helper.HTTP, uri string) (string, error) {
 
 // GetInfo returns the data for the Elasticsearch / endpoint
 func GetInfo(http *helper.HTTP, uri string) (*Info, error) {
-	defer http.SetURI(uri)
-
-	// Parses the uri to replace the path
-	u, _ := url.Parse(uri)
-	u.Path = ""
 
-	// Http helper includes the HostData with username and password
-	http.SetURI(u.String())
-	content, err := http.FetchContent()
+	content, err := fetchPath(http, uri, "/")
 	if err != nil {
 		return nil, err
 	}
@@ -143,6 +136,7 @@ func fetchPath(http *helper.HTTP, uri, path string) ([]byte, error) {
 	// Parses the uri to replace the path
 	u, _ := url.Parse(uri)
 	u.Path = path
+	u.RawQuery = ""
 
 	// Http helper includes the HostData with username and password
 	http.SetURI(u.String())

diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -35,6 +35,7 @@ import (
 	"bytes"
 
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_recovery"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/ml_job"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
@@ -44,6 +45,7 @@ import (
 
 var metricSets = []string{
 	"index",
+	"index_recovery",
 	"index_summary",
 	"ml_job",
 	"node",
@@ -116,6 +118,7 @@ func getConfig(metricset string) map[string]interface{} {
 		"module":     "elasticsearch",
 		"metricsets": []string{metricset},
 		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+		"index_recovery.active_only": false,
 	}
 }
 

diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go
diff --git a/metricbeat/module/elasticsearch/index_recovery/_meta/data.json b/metricbeat/module/elasticsearch/index_recovery/_meta/data.json
@@ -0,0 +1,34 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "elasticsearch": {
+        "index": {
+            "name": "test-index",
+            "recovery": {
+                "id": 0,
+                "primary": true,
+                "source": {},
+                "stage": "DONE",
+                "target": {
+                    "host": "127.0.0.1",
+                    "id": "ODWb5m6zT9q0lf8YTPyWWg",
+                    "name": "ODWb5m6"
+                },
+                "type": "EMPTY_STORE"
+            }
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9200",
+        "module": "elasticsearch",
+        "name": "index_recovery",
+        "namespace": "elasticsearch.index.recovery",
+        "rtt": 115
+    },
+    "service": {
+        "name": "elasticsearch"
+    }
+}
diff --git a/metricbeat/module/elasticsearch/index_recovery/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/index_recovery/_meta/docs.asciidoc
@@ -0,0 +1,15 @@
+This is the index_recovery metricset of the module Elasticsearch.
+
+By default only data about indices which are under active recovery are fetched.
+To gather data about all indices set `index_recovery.active_only: false`:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+- module: elasticsearch
+  metricsets:
+    - index_recovery
+  hosts: ["localhost:9200"]
+  index_recovery.active_only: false
+-------------------------------------------------------------------------------------
+
+
diff --git a/metricbeat/module/elasticsearch/index_recovery/_meta/fields.yml b/metricbeat/module/elasticsearch/index_recovery/_meta/fields.yml
@@ -0,0 +1,47 @@
+- name: index.recovery
+  type: group
+  description: >
+    index
+  fields:
+    - name: id
+      type: long
+      description: >
+        Shard recovery id.
+    - name: type
+      type: keyword
+      description: >
+        Shard recovery type.
+    - name: primary
+      type: boolean
+      description: >
+        True if primary shard.
+    - name: stage
+      type: keyword
+      description: >
+        Recovery stage.
+
+    - name: target.id
+      type: keyword
+      description: >
+        Target node id.
+    - name: target.host
+      type: ip
+      description: >
+        Target node host address.
+    - name: target.name
+      type: keyword
+      description: >
+        Target node name.
+
+    - name: source.id
+      type: keyword
+      description: >
+        Source node id.
+    - name: source.host
+      type: ip
+      description: >
+        Source node host address.
+    - name: source.name
+      type: keyword
+      description: >
+        Source node name.