feat: allow user set forcePathStyle for s3 storage (#731)

warjiang · web-flow · commit 0cfeb2e0080b · 2023-11-03T16:44:33.000+08:00
* feat: update storage lib and export cmd params

Signed-off-by: warjiang &lt;1096409085@qq.com&gt;

* feat: upgrade chartmuseum/storage to v0.14.1

Signed-off-by: warjiang &lt;1096409085@qq.com&gt;

* doc: update usage of force-path-style

Signed-off-by: warjiang &lt;1096409085@qq.com&gt;

---------

Signed-off-by: warjiang &lt;1096409085@qq.com&gt;
diff --git a/README.md b/README.md
@@ -188,6 +188,20 @@ You need at least the following permissions inside your IAM Policy
 In order to work with AWS service accounts you may need to set `AWS_SDK_LOAD_CONFIG=1` in your environment.
 For more context, please see [here](https://github.com/helm/chartmuseum/issues/280#issuecomment-592292527).
 
+If you are using S3-Compatible storage, provider of S3 storage has [disabled path-style and force virtual hosted-style](https://aws.amazon.com/cn/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/), you can use specify `storage-amazon-force-path-style` options as following example:
+```
+export AWS_ACCESS_KEY_ID=""
+export AWS_SECRET_ACCESS_KEY=""
+chartmuseum --debug --port=8080 \
+  --storage="amazon" \
+  --storage-amazon-bucket="my-s3-bucket" \
+  --storage-amazon-prefix="" \
+  --storage-amazon-region="us-east-1" \
+  --storage-amazon-endpoint="my-s3-compatible-service-endpoint"
+  --storage-amazon-force-path-style=false
+```
+
+
 For DigitalOcean, set the credentials using environment variable and pass the `endpoint`.
 Note below, that the region `us-east-1` needs to be set, since that is how the DigitalOcean cli implementation functions. The actual region of your spaces location is defined by the endpoint. Below we are using Frankfurt as an example.
 ```bash
diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go
@@ -176,12 +176,16 @@ func amazonBackendFromConfig(conf *config.Config) storage.Backend {
 		conf.Set("storage.amazon.region", "us-east-1")
 	}
 	crashIfConfigMissingVars(conf, []string{"storage.amazon.bucket", "storage.amazon.region"})
-	return storage.NewAmazonS3Backend(
+	forcePathStyle := conf.GetBool("storage.amazon.forcepathstyle")
+	return storage.NewAmazonS3BackendWithOptions(
 		conf.GetString("storage.amazon.bucket"),
 		conf.GetString("storage.amazon.prefix"),
 		conf.GetString("storage.amazon.region"),
 		conf.GetString("storage.amazon.endpoint"),
 		conf.GetString("storage.amazon.sse"),
+		&storage.AmazonS3Options{
+			S3ForcePathStyle: &forcePathStyle,
+		},
 	)
 }
 
diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/alicebob/miniredis v2.5.0+incompatible
 	github.com/chartmuseum/auth v0.5.0
-	github.com/chartmuseum/storage v0.14.0
+	github.com/chartmuseum/storage v0.14.1
 	github.com/gin-contrib/size v0.0.0-20230212012657-e14a14094dc4
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-redis/redis v6.15.9+incompatible
diff --git a/go.sum b/go.sum
@@ -111,8 +111,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chartmuseum/auth v0.5.0 h1:ENNmoxvjxcR/JR0HrghAEtGQe7hToMNj16+UoS5CK9Y=
 github.com/chartmuseum/auth v0.5.0/go.mod h1:BvoSXHyvbsq+/bbhNgVTDQsModM+HERBTNY5o9Vyrig=
-github.com/chartmuseum/storage v0.14.0 h1:R/Mp4fRaY5HCtLIEJomrsT415kEi6QumertR74cbHZU=
-github.com/chartmuseum/storage v0.14.0/go.mod h1:LB+/k4kZkOu2AW3myaOw/AQt0JtjcNLwL1ktKxoWVug=
+github.com/chartmuseum/storage v0.14.1 h1:Az+YUopt+GjEg4r4kdq59ZMxuwuytaDbRIeWE6tghbU=
+github.com/chartmuseum/storage v0.14.1/go.mod h1:LB+/k4kZkOu2AW3myaOw/AQt0JtjcNLwL1ktKxoWVug=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
diff --git a/pkg/config/vars.go b/pkg/config/vars.go
@@ -355,6 +355,15 @@ var configVars = map[string]configVar{
 			EnvVar: "STORAGE_AMAZON_SSE",
 		},
 	},
+	"storage.amazon.forcepathstyle": {
+		Type:    boolType,
+		Default: true,
+		CLIFlag: cli.StringFlag{
+			Name:   "storage-amazon-force-path-style",
+			Usage:  "whether to force path style for amazon storage backend",
+			EnvVar: "STORAGE_AMAZON_FORCE_PATH_STYLE",
+		},
+	},
 	"storage.google.bucket": {
 		Type:    stringType,
 		Default: "",

Original file line number	Diff line number	Diff line change
`@@ -176,12 +176,16 @@ func amazonBackendFromConfig(conf *config.Config) storage.Backend {`
`176`	`176`	`conf.Set("storage.amazon.region", "us-east-1")`
`177`	`177`	`}`
`178`	`178`	`crashIfConfigMissingVars(conf, []string{"storage.amazon.bucket", "storage.amazon.region"})`
`179`		`- return storage.NewAmazonS3Backend(`
	`179`	`+ forcePathStyle := conf.GetBool("storage.amazon.forcepathstyle")`
	`180`	`+ return storage.NewAmazonS3BackendWithOptions(`
`180`	`181`	`conf.GetString("storage.amazon.bucket"),`
`181`	`182`	`conf.GetString("storage.amazon.prefix"),`
`182`	`183`	`conf.GetString("storage.amazon.region"),`
`183`	`184`	`conf.GetString("storage.amazon.endpoint"),`
`184`	`185`	`conf.GetString("storage.amazon.sse"),`
	`186`	`+ &storage.AmazonS3Options{`
	`187`	`+ S3ForcePathStyle: &forcePathStyle,`
	`188`	`+ },`
`185`	`189`	`)`
`186`	`190`	`}`
`187`	`191`