fix(inputs.cloudwatch): customizable batch size when querying (#10851)

powersj · web-flow · commit 196abb74cf12 · 2022-08-01T13:09:25.000-06:00
diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md
@@ -91,6 +91,12 @@ API endpoint. In the following order the plugin will attempt to authenticate.
   ## Timeout for http requests made by the cloudwatch client.
   # timeout = "5s"
 
+  ## Batch Size
+  ## The size of each batch to send requests to Cloudwatch. 500 is the
+  ## suggested largest size. If a request gets to large (413 errors), consider
+  ## reducing this amount.
+  # batch_size = 500
+
   ## Namespace-wide statistic filters. These allow fewer queries to be made to
   ## cloudwatch.
   # statistic_include = [ "average", "sum", "minimum", "maximum", sample_count" ]
diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go
@@ -55,6 +55,7 @@ type CloudWatch struct {
 	CacheTTL       config.Duration `toml:"cache_ttl"`
 	RateLimit      int             `toml:"ratelimit"`
 	RecentlyActive string          `toml:"recently_active"`
+	BatchSize      int             `toml:"batch_size"`
 
 	Log telegraf.Logger `toml:"-"`
 
@@ -146,12 +147,10 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {
 	results := map[string][]types.MetricDataResult{}
 
 	for namespace, namespacedQueries := range queries {
-		// 500 is the maximum number of metric data queries a `GetMetricData` request can contain.
-		batchSize := 500
 		var batches [][]types.MetricDataQuery
 
-		for batchSize < len(namespacedQueries) {
-			namespacedQueries, batches = namespacedQueries[batchSize:], append(batches, namespacedQueries[0:batchSize:batchSize])
+		for c.BatchSize < len(namespacedQueries) {
+			namespacedQueries, batches = namespacedQueries[c.BatchSize:], append(batches, namespacedQueries[0:c.BatchSize:c.BatchSize])
 		}
 		batches = append(batches, namespacedQueries)
 
@@ -530,6 +529,7 @@ func New() *CloudWatch {
 		CacheTTL:  config.Duration(time.Hour),
 		RateLimit: 25,
 		Timeout:   config.Duration(time.Second * 5),
+		BatchSize: 500,
 	}
 }
 
diff --git a/plugins/inputs/cloudwatch/cloudwatch_test.go b/plugins/inputs/cloudwatch/cloudwatch_test.go
@@ -106,6 +106,7 @@ func TestGather(t *testing.T) {
 		Delay:     internalDuration,
 		Period:    internalDuration,
 		RateLimit: 200,
+		BatchSize: 500,
 	}
 
 	var acc testutil.Accumulator
@@ -137,6 +138,7 @@ func TestGather_MultipleNamespaces(t *testing.T) {
 		Delay:      internalDuration,
 		Period:     internalDuration,
 		RateLimit:  200,
+		BatchSize:  500,
 	}
 
 	var acc testutil.Accumulator
@@ -213,6 +215,7 @@ func TestSelectMetrics(t *testing.T) {
 		Delay:     internalDuration,
 		Period:    internalDuration,
 		RateLimit: 200,
+		BatchSize: 500,
 		Metrics: []*Metric{
 			{
 				MetricNames: []string{"Latency", "RequestCount"},
@@ -258,6 +261,7 @@ func TestGenerateStatisticsInputParams(t *testing.T) {
 		Namespaces: []string{namespace},
 		Delay:      internalDuration,
 		Period:     internalDuration,
+		BatchSize:  500,
 	}
 
 	require.NoError(t, c.initializeCloudWatch())
@@ -297,6 +301,7 @@ func TestGenerateStatisticsInputParamsFiltered(t *testing.T) {
 		Namespaces: []string{namespace},
 		Delay:      internalDuration,
 		Period:     internalDuration,
+		BatchSize:  500,
 	}
 
 	require.NoError(t, c.initializeCloudWatch())
@@ -336,6 +341,7 @@ func TestUpdateWindow(t *testing.T) {
 		Namespace: "AWS/ELB",
 		Delay:     internalDuration,
 		Period:    internalDuration,
+		BatchSize: 500,
 	}
 
 	now := time.Now()
@@ -364,6 +370,7 @@ func TestProxyFunction(t *testing.T) {
 		HTTPProxy: proxy.HTTPProxy{
 			HTTPProxyURL: "http://www.penguins.com",
 		},
+		BatchSize: 500,
 	}
 
 	proxyFunction, err := c.HTTPProxy.Proxy()
@@ -378,7 +385,11 @@ func TestProxyFunction(t *testing.T) {
 }
 
 func TestCombineNamespaces(t *testing.T) {
-	c := &CloudWatch{Namespace: "AWS/ELB", Namespaces: []string{"AWS/EC2", "AWS/Billing"}}
+	c := &CloudWatch{
+		Namespace:  "AWS/ELB",
+		Namespaces: []string{"AWS/EC2", "AWS/Billing"},
+		BatchSize:  500,
+	}
 
 	require.NoError(t, c.Init())
 	require.Equal(t, []string{"AWS/EC2", "AWS/Billing", "AWS/ELB"}, c.Namespaces)
diff --git a/plugins/inputs/cloudwatch/sample.conf b/plugins/inputs/cloudwatch/sample.conf
@@ -72,6 +72,12 @@
   ## Timeout for http requests made by the cloudwatch client.
   # timeout = "5s"
 
+  ## Batch Size
+  ## The size of each batch to send requests to Cloudwatch. 500 is the
+  ## suggested largest size. If a request gets to large (413 errors), consider
+  ## reducing this amount.
+  # batch_size = 500
+
   ## Namespace-wide statistic filters. These allow fewer queries to be made to
   ## cloudwatch.
   # statistic_include = [ "average", "sum", "minimum", "maximum", sample_count" ]
