feat(aws): Support Custom endpoints (#5942)

bbernays · web-flow · commit 746ba6501727 · 2022-12-23T18:34:47.000Z
#### Summary

This PR enables users to configure a single global endpoint that the AWS Plugin will use to sync resources. This enables users to point CloudQuery at 3rd party tools like LocalStack
&lt;!--
diff --git a/plugins/source/aws/client/client.go b/plugins/source/aws/client/client.go
@@ -235,6 +235,18 @@ func configureAwsClient(ctx context.Context, logger zerolog.Logger, awsConfig *S
 			})
 		}),
 	}
+	if awsConfig.EndpointURL != "" {
+		configFns = append(configFns, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...any) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               awsConfig.EndpointURL,
+					HostnameImmutable: aws.ToBool(awsConfig.HostnameImmutable),
+					PartitionID:       awsConfig.PartitionID,
+					SigningRegion:     awsConfig.SigningRegion,
+				}, nil
+			})),
+		)
+	}
 
 	if account.DefaultRegion != "" {
 		// According to the docs: If multiple WithDefaultRegion calls are made, the last call overrides the previous call values
diff --git a/plugins/source/aws/client/spec.go b/plugins/source/aws/client/spec.go
@@ -33,15 +33,31 @@ type AwsOrg struct {
 }
 
 type Spec struct {
-	Regions      []string  `json:"regions,omitempty"`
-	Accounts     []Account `json:"accounts"`
-	Organization *AwsOrg   `json:"org"`
-	AWSDebug     bool      `json:"aws_debug,omitempty"`
-	MaxRetries   *int      `json:"max_retries,omitempty"`
-	MaxBackoff   *int      `json:"max_backoff,omitempty"`
+	Regions           []string  `json:"regions,omitempty"`
+	Accounts          []Account `json:"accounts"`
+	Organization      *AwsOrg   `json:"org"`
+	AWSDebug          bool      `json:"aws_debug,omitempty"`
+	MaxRetries        *int      `json:"max_retries,omitempty"`
+	MaxBackoff        *int      `json:"max_backoff,omitempty"`
+	EndpointURL       string    `json:"custom_endpoint_url,omitempty"`
+	HostnameImmutable *bool     `json:"custom_endpoint_hostname_immutable,omitempty"`
+	PartitionID       string    `json:"custom_endpoint_partition_id,omitempty"`
+	SigningRegion     string    `json:"custom_endpoint_signing_region,omitempty"`
 }
 
 func (s *Spec) Validate() error {
+	if s.EndpointURL != "" {
+		if s.PartitionID == "" {
+			return fmt.Errorf("custom_endpoint_partition_id is required when custom_endpoint_url is set")
+		}
+		if s.SigningRegion == "" {
+			return fmt.Errorf("custom_endpoint_signing_region is required when custom_endpoint_url is set")
+		}
+		if s.HostnameImmutable == nil {
+			return fmt.Errorf("custom_endpoint_hostname_immutable is required when custom_endpoint_url is set")
+		}
+	}
+
 	if s.Organization != nil && len(s.Accounts) > 0 {
 		return errors.New("specifying accounts via both the Accounts and Org properties is not supported. To achieve both, use multiple source configurations")
 	}
diff --git a/website/pages/docs/plugins/sources/aws/configuration.md b/website/pages/docs/plugins/sources/aws/configuration.md
@@ -100,6 +100,31 @@ This is the (nested) spec used by the AWS source plugin.
 
   If true, will log AWS debug logs, including retries and other request/response metadata
 
+- `max_retries` (int) (default: 10)
+
+  Defines the maximum number of times an API request will be retried 
+
+- `max_retries` (int) (max_backoff: 30)
+  
+  Defines the duration between retry attempts
+
+- `custom_endpoint_url` (string) (default: not used)
+
+  The base URL endpoint the SDK API clients will use to make API calls to. The SDK will suffix URI path and query elements to this endpoint
+
+- `custom_endpoint_hostname_immutable` (bool) (default: not used)
+
+  Specifies if the endpoint's hostname can be modified by the SDK's API client. When using something like LocalStack make sure to set it equal to `True`
+
+- `custom_endpoint_partition_id` (string) (default: not used)
+
+  The AWS partition the endpoint belongs to
+
+- `custom_endpoint_signing_region` (string) (default: not used)
+
+  The region that should be used for signing the request to the endpoint
+
+
 ## accounts
 
 This is used to specify one or more accounts to extract information from. Note that it should be an array of objects, each with the following fields:
