Make API address and Shard ID required in Cloud Foundry settings (#21759)

jsoriano · web-flow · commit 3e5a16780921 · 2020-10-14T12:58:48.000+02:00
Having an auto-generated Shard ID leads to duplicated data when trying
to scale, increasing the problems of loaded systems. Forcing to set a shard ID
makes the user more conscious of the implications of this setting.

API address should be always set in a real deployment.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -25,6 +25,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
 - Update to Golang 1.12.1. {pull}11330[11330]
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
+- API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
 
 *Auditbeat*
 
@@ -78,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
+- API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
 
 *Heartbeat*
 
@@ -95,6 +97,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix ECS compliance of user.id field in system/users  metricset {pull}19019[19019]
 - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718]
 - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457]
+- API address and shard ID are required settings in the Cloud Foundry module. {pull}21759[21759]
 
 *Packetbeat*
 
diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml
@@ -11,7 +11,7 @@ filebeat.inputs:
     #doppler_address: ${DOPPLER_ADDRESS}
     #uaa_address: ${UAA_ADDRESS}
     #rlp_address: ${RLP_ADDRESS}
-    #shard_id: ${SHARD_ID}
+    shard_id: ${SHARD_ID}
     #version: v1
 
 
diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml
@@ -11,7 +11,8 @@ metricbeat.modules:
     #doppler_address: ${DOPPLER_ADDRESS}
     #uaa_address: ${UAA_ADDRESS}
     #rlp_address: ${RLP_ADDRESS}
-    #shard_id: ${SHARD_ID}
+    shard_id: ${SHARD_ID}
+    #version: v1
 
 
 #================================ Outputs =====================================
diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc
@@ -71,11 +71,3 @@ filebeat   started           1/1         512M     1G
 
 Log events should start flowing to Elasticsearch. The events are annotated with
 metadata added by the <<add-cloudfoundry-metadata>> processor.
-
-
-[WARNING]
-=======================================
-*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that
-{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id`
-will result in duplicate events being pushed to Elasticsearch.
-=======================================
diff --git a/metricbeat/docs/modules/cloudfoundry.asciidoc b/metricbeat/docs/modules/cloudfoundry.asciidoc
@@ -117,7 +117,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "".
 === `shard_id`
 
 Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events
-from the RLP Gateway. Default: "(generated UUID)".
+from the RLP Gateway.
 
 [float]
 ==== `version`
@@ -152,6 +152,7 @@ metricbeat.modules:
   rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}'
   client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}'
   client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'
+  shard_id: metricbeat
   version: v1
 ----
 
diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc
@@ -71,11 +71,3 @@ metricbeat   started           1/1         512M     1G
 
 Metrics should start flowing to Elasticsearch. The events are annotated with
 metadata added by the <<add-cloudfoundry-metadata>> processor.
-
-
-[WARNING]
-=======================================
-*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that
-{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id`
-will result in duplicate events being pushed to Elasticsearch.
-=======================================
diff --git a/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc b/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc
@@ -23,6 +23,7 @@ Example configurations:
   api_address: https://api.dev.cfdev.sh
   client_id: uaa-filebeat
   client_secret: verysecret
+  shard_id: filebeat
   ssl:
       verification_mode: none
 ----
@@ -34,6 +35,7 @@ Example configurations:
   api_address: https://api.dev.cfdev.sh
   client_id: uaa-filebeat
   client_secret: verysecret
+  shard_id: filebeat
   ssl.certificate_authorities: ["/etc/pki/cf/ca.pem"]
   ssl.certificate: "/etc/pki/cf/cert.pem"
   ssl.key: "/etc/pki/cf/cert.key"
diff --git a/x-pack/libbeat/common/cloudfoundry/config.go b/x-pack/libbeat/common/cloudfoundry/config.go
@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gofrs/uuid"
-
 	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
 )
 
@@ -32,14 +30,14 @@ type Config struct {
 	TLS *tlscommon.Config `config:"ssl"`
 
 	// Override URLs returned from the CF client
-	APIAddress     string `config:"api_address"`
+	APIAddress     string `config:"api_address" validate:"required"`
 	DopplerAddress string `config:"doppler_address"`
 	UaaAddress     string `config:"uaa_address"`
 	RlpAddress     string `config:"rlp_address"`
 
 	// ShardID when retrieving events from loggregator, sharing this ID across
 	// multiple filebeats will shard the load of receiving and sending events.
-	ShardID string `config:"shard_id"`
+	ShardID string `config:"shard_id" validate:"required"`
 
 	// Maximum amount of time to cache application objects from CF client.
 	CacheDuration time.Duration `config:"cache_duration"`
@@ -50,13 +48,6 @@ type Config struct {
 
 // InitDefaults initialize the defaults for the configuration.
 func (c *Config) InitDefaults() {
-	// If not provided by the user; subscription ID should be a unique string to avoid clustering by default.
-	// Default to using a UUID4 string.
-	uuid, err := uuid.NewV4()
-	if err != nil {
-		panic(err)
-	}
-	c.ShardID = uuid.String()
 	c.CacheDuration = 120 * time.Second
 	c.CacheRetryDelay = 20 * time.Second
 	c.Version = ConsumerVersionV1
diff --git a/x-pack/libbeat/common/cloudfoundry/config_test.go b/x-pack/libbeat/common/cloudfoundry/config_test.go
@@ -22,26 +22,48 @@ func TestValidation(t *testing.T) {
 
 	var noId Config
 	assert.Error(t, ucfg.MustNewFrom(common.MapStr{
+		"api_address":   "https://api.dev.cfdev.sh",
 		"client_secret": "client_secret",
+		"shard_id":      "beats-test-1",
 	}).Unpack(&noId))
 
 	var noSecret Config
 	assert.Error(t, ucfg.MustNewFrom(common.MapStr{
-		"client_id": "client_id",
+		"api_address": "https://api.dev.cfdev.sh",
+		"client_id":   "client_id",
+		"shard_id":    "beats-test-1",
 	}).Unpack(&noSecret))
 
+	var noAPI Config
+	assert.Error(t, ucfg.MustNewFrom(common.MapStr{
+		"client_id":     "client_id",
+		"client_secret": "client_secret",
+		"shard_id":      "beats-test-1",
+	}).Unpack(&noAPI))
+
+	var noShardID Config
+	assert.Error(t, ucfg.MustNewFrom(common.MapStr{
+		"api_address":   "https://api.dev.cfdev.sh",
+		"client_id":     "client_id",
+		"client_secret": "client_secret",
+	}).Unpack(&noShardID))
+
 	var valid Config
 	assert.NoError(t, ucfg.MustNewFrom(common.MapStr{
+		"api_address":   "https://api.dev.cfdev.sh",
 		"client_id":     "client_id",
 		"client_secret": "client_secret",
+		"shard_id":      "beats-test-1",
 	}).Unpack(&valid))
 }
 
 func TestInitDefaults(t *testing.T) {
 	var cfCfg Config
 	assert.NoError(t, ucfg.MustNewFrom(common.MapStr{
+		"api_address":   "https://api.dev.cfdev.sh",
 		"client_id":     "client_id",
 		"client_secret": "client_secret",
+		"shard_id":      "beats-test-1",
 	}).Unpack(&cfCfg))
-	assert.Len(t, cfCfg.ShardID, 36)
+	assert.Equal(t, ConsumerVersionV1, cfCfg.Version)
 }
diff --git a/x-pack/libbeat/common/cloudfoundry/test/config.go b/x-pack/libbeat/common/cloudfoundry/test/config.go
@@ -7,23 +7,30 @@ package test
 import (
 	"os"
 	"testing"
+
+	"github.com/gofrs/uuid"
 )
 
 func GetConfigFromEnv(t *testing.T) map[string]interface{} {
 	t.Helper()
 
+	shardID, err := uuid.NewV4()
+	if err != nil {
+		t.Fatalf("Unable to create a random shard ID: %v", err)
+	}
+
 	config := map[string]interface{}{
 		"api_address":   lookupEnv(t, "CLOUDFOUNDRY_API_ADDRESS"),
 		"client_id":     lookupEnv(t, "CLOUDFOUNDRY_CLIENT_ID"),
 		"client_secret": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_SECRET"),
+		"shard_id":      shardID.String(),
 
 		"ssl.verification_mode": "none",
 	}
 
 	optionalConfig(config, "uaa_address", "CLOUDFOUNDRY_UAA_ADDRESS")
 	optionalConfig(config, "rlp_address", "CLOUDFOUNDRY_RLP_ADDRESS")
 	optionalConfig(config, "doppler_address", "CLOUDFOUNDRY_DOPPLER_ADDRESS")
-	optionalConfig(config, "shard_id", "CLOUDFOUNDRY_SHARD_ID")
 
 	if t.Failed() {
 		t.FailNow()
diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go
@@ -5,6 +5,7 @@
 package add_cloudfoundry_metadata
 
 import (
+	"github.com/gofrs/uuid"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
@@ -32,6 +33,11 @@ const selector = "add_cloudfoundry_metadata"
 // New constructs a new add_cloudfoundry_metadata processor.
 func New(cfg *common.Config) (processors.Processor, error) {
 	var config cloudfoundry.Config
+
+	// ShardID is required in cloudfoundry config to consume from the firehose,
+	// but not for metadata requests, randomly generate one and use it.
+	config.ShardID = uuid.Must(uuid.NewV4()).String()
+
 	if err := cfg.Unpack(&config); err != nil {
 		return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName)
 	}
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -398,6 +398,7 @@ metricbeat.modules:
   rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}'
   client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}'
   client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'
+  shard_id: metricbeat
   version: v1
 
 #----------------------------- CockroachDB Module -----------------------------
diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml
@@ -10,4 +10,5 @@
   rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}'
   client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}'
   client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'
+  shard_id: metricbeat
   version: v1
diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml
@@ -7,3 +7,4 @@
   api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}'
   client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}'
   client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'
+  shard_id: metricbeat
diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc
@@ -107,7 +107,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "".
 === `shard_id`
 
 Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events
-from the RLP Gateway. Default: "(generated UUID)".
+from the RLP Gateway.
 
 [float]
 ==== `version`
diff --git a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled
@@ -10,3 +10,4 @@
   api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}'
   client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}'
   client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'
+  shard_id: metricbeat
