Skip to content

Commit ba552f6

Browse files
author
kaiyan-sheng
authored
Deprecate aws_partition and use endpoint,regions instead (#23539)
1 parent e5cd64f commit ba552f6

7 files changed

Lines changed: 77 additions & 9 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
582582
- Honor kube event resysncs to handle missed watch events {pull}22668[22668]
583583
- Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324]
584584
- Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330]
585+
- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539]
585586
- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
586587
- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
587588

‎metricbeat/docs/modules/aws.asciidoc‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Please see <<aws-credentials-config,AWS credentials options>> for more details.
3131
This module also accepts optional configuration `regions` to specify which
3232
AWS regions to query metrics from. If the `regions` parameter is not set in the
3333
config file, then by default, the `aws` module will query metrics from all available
34-
AWS regions.
34+
AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter.
3535

3636
* *latency*
3737

@@ -40,6 +40,27 @@ process larger than Metricbeat collection period. This case, please specify a
4040
`latency` parameter so collection start time and end time will be shifted by the
4141
given latency amount.
4242

43+
* *endpoint*
44+
45+
Most AWS services offer a regional endpoint that can be used to make requests.
46+
The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
47+
Some services, such as IAM, do not support regions. The endpoints for these
48+
services do not include a region. In `aws` module, `endpoint` config is to set
49+
the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
50+
`sc2s.sgov.gov`.
51+
52+
If endpoint is specified, `regions` config becomes required. For example:
53+
54+
[source,yaml]
55+
----
56+
- module: aws
57+
period: 5m
58+
endpoint: amazonaws.com.cn
59+
regions: cn-north-1
60+
metricsets:
61+
- ec2
62+
----
63+
4364
The aws module comes with a predefined dashboard. For example:
4465

4566
image::./images/metricbeat-aws-overview.png[]

‎x-pack/filebeat/input/awss3/input.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C
113113

114114
log.Debug("s3 service name = ", s3Servicename)
115115
log.Debug("s3 input config max_number_of_messages = ", in.config.MaxNumberOfMessages)
116+
log.Debug("s3 input config endpoint = ", in.config.AwsConfig.Endpoint)
116117
return &s3Collector{
117118
cancellation: ctxtool.FromCanceller(ctx.Cancelation),
118119
logger: log,

‎x-pack/libbeat/common/aws/credentials.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/aws/aws-sdk-go-v2/service/sts"
1313
"github.com/pkg/errors"
1414

15+
"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
1516
"github.com/elastic/beats/v7/libbeat/logp"
1617
)
1718

@@ -24,7 +25,7 @@ type ConfigAWS struct {
2425
SharedCredentialFile string `config:"shared_credential_file"`
2526
Endpoint string `config:"endpoint"`
2627
RoleArn string `config:"role_arn"`
27-
AWSPartition string `config:"aws_partition"`
28+
AWSPartition string `config:"aws_partition"` // Deprecated.
2829
}
2930

3031
// GetAWSCredentials function gets aws credentials from the config.
@@ -104,3 +105,11 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName
104105
}
105106
return awsConfig
106107
}
108+
109+
// Validate checks for deprecated config option
110+
func (c ConfigAWS) Validate() error {
111+
if c.AWSPartition != "" {
112+
cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.")
113+
}
114+
return nil
115+
}

‎x-pack/libbeat/docs/aws-credentials-config.asciidoc‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,14 @@ To configure AWS credentials, either put the credentials into the {beatname_uc}
99
* *session_token*: required when using temporary security credentials.
1010
* *credential_profile_name*: profile name in shared credentials file.
1111
* *shared_credential_file*: directory of the shared credentials file.
12-
* *endpoint*: URL of the entry point for an AWS web service.
1312
* *role_arn*: AWS IAM Role to assume.
14-
* *aws_partition*: AWS region parttion name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`.
13+
* *endpoint*: URL of the entry point for an AWS web service.
14+
Most AWS services offer a regional endpoint that can be used to make requests.
15+
The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
16+
Some services, such as IAM, do not support regions. The endpoints for these
17+
services do not include a region. In `aws` module, `endpoint` config is to set
18+
the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
19+
`sc2s.sgov.gov`.
1520

1621
[float]
1722
==== Supported Formats

‎x-pack/metricbeat/module/aws/_meta/docs.asciidoc‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ Please see <<aws-credentials-config,AWS credentials options>> for more details.
2323
This module also accepts optional configuration `regions` to specify which
2424
AWS regions to query metrics from. If the `regions` parameter is not set in the
2525
config file, then by default, the `aws` module will query metrics from all available
26-
AWS regions.
26+
AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter.
2727

2828
* *latency*
2929

@@ -32,6 +32,27 @@ process larger than Metricbeat collection period. This case, please specify a
3232
`latency` parameter so collection start time and end time will be shifted by the
3333
given latency amount.
3434

35+
* *endpoint*
36+
37+
Most AWS services offer a regional endpoint that can be used to make requests.
38+
The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
39+
Some services, such as IAM, do not support regions. The endpoints for these
40+
services do not include a region. In `aws` module, `endpoint` config is to set
41+
the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
42+
`sc2s.sgov.gov`.
43+
44+
If endpoint is specified, `regions` config becomes required. For example:
45+
46+
[source,yaml]
47+
----
48+
- module: aws
49+
period: 5m
50+
endpoint: amazonaws.com.cn
51+
regions: cn-north-1
52+
metricsets:
53+
- ec2
54+
----
55+
3556
The aws module comes with a predefined dashboard. For example:
3657

3758
image::./images/metricbeat-aws-overview.png[]

‎x-pack/metricbeat/module/aws/aws.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package aws
66

77
import (
88
"context"
9+
"fmt"
910
"time"
1011

1112
awssdk "github.com/aws/aws-sdk-go-v2/aws"
@@ -16,7 +17,6 @@ import (
1617
"github.com/aws/aws-sdk-go-v2/service/rds"
1718
"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
1819
"github.com/aws/aws-sdk-go-v2/service/sts"
19-
"github.com/pkg/errors"
2020

2121
"github.com/elastic/beats/v7/libbeat/common"
2222
"github.com/elastic/beats/v7/metricbeat/mb"
@@ -78,20 +78,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
7878

7979
awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig)
8080
if err != nil {
81-
return nil, errors.Wrap(err, "failed to get aws credentials, please check AWS credential in config")
81+
return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err)
8282
}
8383

8484
_, err = awsConfig.Credentials.Retrieve()
8585
if err != nil {
86-
return nil, errors.Wrap(err, "failed to retrieve aws credentials, please check AWS credential in config")
86+
return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err)
8787
}
8888

89+
base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint)
8990
metricSet := MetricSet{
9091
BaseMetricSet: base,
9192
Period: config.Period,
9293
Latency: config.Latency,
9394
AwsConfig: &awsConfig,
9495
TagsFilter: config.TagsFilter,
96+
Endpoint: config.AWSConfig.Endpoint,
9597
}
9698

9799
base.Logger().Debug("Metricset level config for period: ", metricSet.Period)
@@ -100,6 +102,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
100102

101103
// Get IAM account name, set region by aws_partition, default is aws global partition
102104
// refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
105+
if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" {
106+
base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.")
107+
}
103108
switch config.AWSConfig.AWSPartition {
104109
case "aws-cn":
105110
awsConfig.Region = "cn-north-1"
@@ -109,6 +114,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
109114
awsConfig.Region = "us-east-1"
110115
}
111116

117+
// If regions in config is not empty, then overwrite the awsConfig.Region
118+
if len(config.Regions) > 0 {
119+
awsConfig.Region = config.Regions[0]
120+
}
121+
112122
// Get IAM account id
113123
svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint(
114124
config.AWSConfig.Endpoint, "sts", "", awsConfig))
@@ -151,7 +161,7 @@ func getRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error
151161
req := svc.DescribeRegionsRequest(input)
152162
output, err := req.Send(context.TODO())
153163
if err != nil {
154-
err = errors.Wrap(err, "Failed DescribeRegions")
164+
err = fmt.Errorf("failed DescribeRegions: %w", err)
155165
return
156166
}
157167
for _, region := range output.Regions {

0 commit comments

Comments
 (0)