Skip to content

Commit adbb1a3

Browse files
author
kaiyan-sheng
authored
Migrate sqs metricset to use cloudwatch input as light weight module (#26117)
1 parent 38e6c59 commit adbb1a3

12 files changed

Lines changed: 144 additions & 306 deletions

File tree

CHANGELOG.next.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -939,6 +939,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
939939
- Add additional network metrics to docker/network {pull}25354[25354]
940940
- Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
941941
- Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
942+
- Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
942943

943944
*Packetbeat*
944945

metricbeat/docs/modules/aws/sqs.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
88

99
include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc[]
1010

11+
This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
1112

1213
==== Fields
1314

x-pack/metricbeat/include/list.go

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

x-pack/metricbeat/module/aws/cloudwatch/metadata.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,22 @@ import (
99

1010
"github.com/elastic/beats/v7/metricbeat/mb"
1111
"github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch/ec2"
12+
"github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch/sqs"
1213
)
1314

1415
// AWS namespaces
1516
const (
1617
namespaceEC2 = "AWS/EC2"
18+
namespaceSQS = "AWS/SQS"
1719
)
1820

1921
// addMetadata adds metadata to the given events map based on namespace
2022
func addMetadata(namespace string, endpoint string, regionName string, awsConfig awssdk.Config, events map[string]mb.Event) map[string]mb.Event {
2123
switch namespace {
2224
case namespaceEC2:
2325
return ec2.AddMetadata(endpoint, regionName, awsConfig, events)
26+
case namespaceSQS:
27+
return sqs.AddMetadata(endpoint, regionName, awsConfig, events)
2428
default:
2529
return events
2630
}
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package sqs
6+
7+
import (
8+
"context"
9+
"fmt"
10+
"strings"
11+
12+
awssdk "github.com/aws/aws-sdk-go-v2/aws"
13+
"github.com/aws/aws-sdk-go-v2/service/sqs"
14+
"github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface"
15+
"github.com/pkg/errors"
16+
17+
"github.com/elastic/beats/v7/libbeat/logp"
18+
"github.com/elastic/beats/v7/metricbeat/mb"
19+
awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
20+
)
21+
22+
const metadataPrefix = "aws.sqs.queue"
23+
24+
// AddMetadata adds metadata for SQS queues from a specific region
25+
func AddMetadata(endpoint string, regionName string, awsConfig awssdk.Config, events map[string]mb.Event) map[string]mb.Event {
26+
svc := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(
27+
endpoint, "sqs", regionName, awsConfig))
28+
29+
// Get queueUrls for each region
30+
queueURLs, err := getQueueUrls(svc)
31+
if err != nil {
32+
logp.Error(fmt.Errorf("getQueueUrls failed, skipping region %s: %w", regionName, err))
33+
return events
34+
}
35+
36+
// collect monitoring state for each instance
37+
for _, queueURL := range queueURLs {
38+
queueURLParsed := strings.Split(queueURL, "/")
39+
queueName := queueURLParsed[len(queueURLParsed)-1]
40+
if _, ok := events[queueName]; !ok {
41+
continue
42+
}
43+
events[queueName].RootFields.Put(metadataPrefix+"name", queueName)
44+
}
45+
return events
46+
}
47+
48+
func getQueueUrls(svc sqsiface.ClientAPI) ([]string, error) {
49+
// ListQueues
50+
listQueuesInput := &sqs.ListQueuesInput{}
51+
req := svc.ListQueuesRequest(listQueuesInput)
52+
output, err := req.Send(context.TODO())
53+
if err != nil {
54+
err = errors.Wrap(err, "Error ListQueues")
55+
return nil, err
56+
}
57+
return output.QueueUrls, nil
58+
}

x-pack/metricbeat/module/aws/module.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ metricsets:
33
- ec2
44
- elb
55
- ebs
6+
- sqs
67
- usage
78
- sns
89
- s3_daily_storage

x-pack/metricbeat/module/aws/sqs/_meta/data.json

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,41 @@
11
{
22
"@timestamp": "2017-10-12T08:05:34.853Z",
33
"aws": {
4+
"cloudwatch": {
5+
"namespace": "AWS/SQS"
6+
},
7+
"dimensions": {
8+
"QueueName": "filebeat-aws-elb-test"
9+
},
410
"sqs": {
511
"empty_receives": 0,
612
"messages": {
713
"delayed": 0,
814
"deleted": 0,
915
"not_visible": 0,
1016
"received": 0,
11-
"sent": 0,
12-
"visible": 2
17+
"sent": 0.2857142857142857,
18+
"visible": 827.8
1319
},
1420
"oldest_message_age": {
15-
"sec": 78494
21+
"sec": 345606.4
1622
},
17-
"queue": {
18-
"name": "test-s3-notification"
19-
},
20-
"sent_message_size": {}
23+
"queuename": "filebeat-aws-elb-test",
24+
"sent_message_size": {
25+
"bytes": 1006.5
26+
}
27+
},
28+
"tags": {
29+
"created-by": "kaiyan"
2130
}
2231
},
2332
"cloud": {
2433
"account": {
25-
"id": "627959692251",
26-
"name": "elastic-test"
34+
"id": "428152502467",
35+
"name": "elastic-beats"
2736
},
2837
"provider": "aws",
29-
"region": "ap-southeast-1"
38+
"region": "eu-central-1"
3039
},
3140
"event": {
3241
"dataset": "aws.sqs",

x-pack/metricbeat/module/aws/sqs/data.go

Lines changed: 0 additions & 31 deletions
This file was deleted.
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
default: true
2+
input:
3+
module: aws
4+
metricset: cloudwatch
5+
defaults:
6+
metrics:
7+
- namespace: AWS/SQS
8+
resource_type: sqs
9+
statistic: ["Average"]
10+
name:
11+
- ApproximateAgeOfOldestMessage
12+
- ApproximateNumberOfMessagesDelayed
13+
- ApproximateNumberOfMessagesNotVisible
14+
- ApproximateNumberOfMessagesVisible
15+
- NumberOfMessagesDeleted
16+
- NumberOfMessagesReceived
17+
- NumberOfMessagesSent
18+
- NumberOfEmptyReceives
19+
- SentMessageSize
20+
processors:
21+
- rename:
22+
ignore_missing: true
23+
fields:
24+
- from: "aws.sqs.metrics.ApproximateAgeOfOldestMessage.avg"
25+
to: "aws.sqs.oldest_message_age.sec"
26+
- from: "aws.sqs.metrics.ApproximateNumberOfMessagesDelayed.avg"
27+
to: "aws.sqs.messages.delayed"
28+
- from: "aws.sqs.metrics.ApproximateNumberOfMessagesNotVisible.avg"
29+
to: "aws.sqs.messages.not_visible"
30+
- from: "aws.sqs.metrics.ApproximateNumberOfMessagesVisible.avg"
31+
to: "aws.sqs.messages.visible"
32+
- from: "aws.sqs.metrics.NumberOfMessagesDeleted.avg"
33+
to: "aws.sqs.messages.deleted"
34+
- from: "aws.sqs.metrics.NumberOfMessagesReceived.avg"
35+
to: "aws.sqs.messages.received"
36+
- from: "aws.sqs.metrics.NumberOfMessagesSent.avg"
37+
to: "aws.sqs.messages.sent"
38+
- from: "aws.sqs.metrics.NumberOfEmptyReceives.avg"
39+
to: "aws.sqs.empty_receives"
40+
- from: "aws.sqs.metrics.SentMessageSize.avg"
41+
to: "aws.sqs.sent_message_size.bytes"
42+
43+
- drop_fields:
44+
ignore_missing: true
45+
fields:
46+
- "aws.sqs.metrics"

0 commit comments

Comments
 (0)