feat: Enable Users to Disable TLS Verification when using Custom Endpoint (#14192)

bbernays · web-flow · commit 1762b3a3197d · 2023-10-05T18:25:00.000Z
#### Summary

&lt;!--
Explain what problem this PR addresses
--&gt;

&lt;!--
diff --git a/plugins/destination/s3/client/client.go b/plugins/destination/s3/client/client.go
@@ -3,11 +3,14 @@ package client
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -62,6 +65,12 @@ func New(ctx context.Context, logger zerolog.Logger, spec []byte, opts plugin.Ne
 
 	cfg.Region = c.spec.Region
 
+	cfg.HTTPClient = awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) {
+		if tr.TLSClientConfig == nil {
+			tr.TLSClientConfig = &tls.Config{}
+		}
+		tr.TLSClientConfig.InsecureSkipVerify = c.spec.EndpointSkipTLSVerify
+	})
 	c.s3Client = s3.NewFromConfig(cfg, func(o *s3.Options) {
 		if len(c.spec.Endpoint) > 0 {
 			baseEndpoint := c.spec.Endpoint
diff --git a/plugins/destination/s3/client/spec.go b/plugins/destination/s3/client/spec.go
@@ -19,12 +19,12 @@ type Spec struct {
 	Athena    bool   `json:"athena,omitempty"`
 	TestWrite *bool  `json:"test_write,omitempty"`
 
-	Endpoint     string `json:"endpoint,omitempty"`
-	UsePathStyle bool   `json:"use_path_style,omitempty"`
-
-	BatchSize      *int64               `json:"batch_size"`
-	BatchSizeBytes *int64               `json:"batch_size_bytes"`
-	BatchTimeout   *configtype.Duration `json:"batch_timeout"`
+	Endpoint              string               `json:"endpoint,omitempty"`
+	UsePathStyle          bool                 `json:"use_path_style,omitempty"`
+	EndpointSkipTLSVerify bool                 `json:"endpoint_skip_tls_verify,omitempty"`
+	BatchSize             *int64               `json:"batch_size"`
+	BatchSizeBytes        *int64               `json:"batch_size_bytes"`
+	BatchTimeout          *configtype.Duration `json:"batch_timeout"`
 }
 
 func (s *Spec) SetDefaults() {
diff --git a/website/pages/docs/plugins/destinations/s3/_configuration.mdx b/website/pages/docs/plugins/destinations/s3/_configuration.mdx
@@ -24,6 +24,7 @@ spec:
     # athena: false # <- set this to true for Athena compatibility
     # test_write: true # tests the ability to write to the bucket before processing the data
     # endpoint: "" # Endpoint to use for S3 API calls.
+    # endpoint_skip_tls_verify # Disable TLS verification if using an untrusted certificate
     # use_path_style: false
     # batch_size: 10000 # 10K entries
     # batch_size_bytes: 52428800 # 50 MiB
diff --git a/website/pages/docs/plugins/destinations/s3/overview.mdx b/website/pages/docs/plugins/destinations/s3/overview.mdx
@@ -82,6 +82,10 @@ This is the (nested) spec used by the CSV destination Plugin.
   Endpoint to use for S3 API calls. This is useful for S3-compatible storage services such as MinIO.
   Note: if you want to use path-style addressing, i.e., `https://s3.amazonaws.com/BUCKET/KEY`, `use_path_style` should be enabled, too.
 
+- `endpoint_skip_tls_verify` (`boolean`) (optional) (default: `false`)
+  
+  Disable TLS verification for requests to your S3 endpoint. This option is intended to be used when using a custom endpoint using the `endpoint` option.
+
 - `use_path_style` (`boolean`) (optional) (default: `false`)
 
   Allows to use path-style addressing in the `endpoint` option, i.e., `https://s3.amazonaws.com/BUCKET/KEY`.
