cloud: add version gate for auth via assume role in AWS and GCP storage and KMS

Rui Hu · Rui Hu · commit cdb0bf456483 · 2022-09-26T16:50:42.000-04:00
Add a version gate for auth via assume role in AWS and GCP storage and KMS to
prevent this type of auth until all nodes in the cluster has been upgraded to
22.2. The gate prevents a class of job failures where sometimes a job can
succeed with assume role auth if its processors happen to all be on 22.2 nodes,
but fail at times when one of its processor nodes don't support assume role.
This version gate preempts the issue by preventing this type of auth until the
cluster has been finalized on 22.2 and gives a better error message of why the
auth cannot be used.

It's important to note that this gate does not prevent a user from creating
a BACKUP job that uses assume role auth, e.g. via the DETACHED option, because
the destination storage is not accessed during planning. This is inline with
existing behavior for other types of auth errors, e.g. if the user enters
incorrect credentials. The BACKUP job will still fail with the version gate
error when it eventually executes.

Release note: None
diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt
@@ -296,4 +296,4 @@ trace.jaeger.agent	string		the address of a Jaeger agent to receive traces using
 trace.opentelemetry.collector	string		address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.
 trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
-version	version	1000022.1-72	set the active cluster version in the format '<major>.<minor>'
+version	version	1000022.1-74	set the active cluster version in the format '<major>.<minor>'
diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html
@@ -230,6 +230,6 @@
 <tr><td><code>trace.opentelemetry.collector</code></td><td>string</td><td><code></code></td><td>address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.</td></tr>
 <tr><td><code>trace.span_registry.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://<ui>/#/debug/tracez</td></tr>
 <tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.</td></tr>
-<tr><td><code>version</code></td><td>version</td><td><code>1000022.1-72</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
+<tr><td><code>version</code></td><td>version</td><td><code>1000022.1-74</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
 </tbody>
 </table>
diff --git a/pkg/cloud/amazon/BUILD.bazel b/pkg/cloud/amazon/BUILD.bazel
@@ -18,6 +18,7 @@ go_library(
         "//pkg/cloud/externalconn",
         "//pkg/cloud/externalconn/connectionpb",
         "//pkg/cloud/externalconn/utils",
+        "//pkg/clusterversion",
         "//pkg/security/username",
         "//pkg/server/telemetry",
         "//pkg/settings",
diff --git a/pkg/cloud/amazon/aws_kms.go b/pkg/cloud/amazon/aws_kms.go
@@ -21,6 +21,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/kms"
 	"github.com/cockroachdb/cockroach/pkg/cloud"
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/errors"
 )
 
@@ -161,6 +162,10 @@ func MakeAWSKMS(ctx context.Context, uri string, env cloud.KMSEnv) (cloud.KMS, e
 	}
 
 	if kmsURIParams.roleARN != "" {
+		if !env.ClusterSettings().Version.IsActive(ctx, clusterversion.SupportAssumeRoleAuth) {
+			return nil, errors.New("cannot authenticate to KMS via assume role until cluster has fully upgraded to 22.2")
+		}
+
 		// If there are delegate roles in the assume-role chain, we create a session
 		// for each role in order for it to fetch the credentials from the next role
 		// in the chain.
diff --git a/pkg/cloud/amazon/s3_storage.go b/pkg/cloud/amazon/s3_storage.go
@@ -33,6 +33,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/cloud"
 	"github.com/cockroachdb/cockroach/pkg/cloud/cloudpb"
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/server/telemetry"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
@@ -470,6 +471,10 @@ func newClient(
 	}
 
 	if conf.roleARN != "" {
+		if !settings.Version.IsActive(ctx, clusterversion.SupportAssumeRoleAuth) {
+			return s3Client{}, "", errors.New("cannot authenticate to cloud storage via assume role until cluster has fully upgraded to 22.2")
+		}
+
 		for _, role := range conf.delegateRoleARNs {
 			intermediateCreds := stscreds.NewCredentials(sess, role)
 			opts.Config.Credentials = intermediateCreds
diff --git a/pkg/cloud/gcp/BUILD.bazel b/pkg/cloud/gcp/BUILD.bazel
@@ -19,6 +19,7 @@ go_library(
         "//pkg/cloud/externalconn",
         "//pkg/cloud/externalconn/connectionpb",
         "//pkg/cloud/externalconn/utils",
+        "//pkg/clusterversion",
         "//pkg/security/username",
         "//pkg/server/telemetry",
         "//pkg/settings",
diff --git a/pkg/cloud/gcp/gcp_kms.go b/pkg/cloud/gcp/gcp_kms.go
@@ -18,6 +18,7 @@ import (
 
 	kms "cloud.google.com/go/kms/apiv1"
 	"github.com/cockroachdb/cockroach/pkg/cloud"
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/errors"
 	"google.golang.org/api/option"
 	kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
@@ -126,6 +127,10 @@ func MakeGCSKMS(ctx context.Context, uri string, env cloud.KMSEnv) (cloud.KMS, e
 	if kmsURIParams.assumeRole == "" {
 		opts = append(opts, credentialsOpt...)
 	} else {
+		if !env.ClusterSettings().Version.IsActive(ctx, clusterversion.SupportAssumeRoleAuth) {
+			return nil, errors.New("cannot authenticate to KMS via assume role until cluster has fully upgraded to 22.2")
+		}
+
 		assumeOpt, err := createImpersonateCredentials(ctx, kmsURIParams.assumeRole, kmsURIParams.delegateRoles, kms.DefaultAuthScopes(), credentialsOpt...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to assume role")
diff --git a/pkg/cloud/gcp/gcs_storage.go b/pkg/cloud/gcp/gcs_storage.go
@@ -24,6 +24,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/cloud"
 	"github.com/cockroachdb/cockroach/pkg/cloud/cloudpb"
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/server/telemetry"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
@@ -181,6 +182,10 @@ func makeGCSStorage(
 	if conf.AssumeRole == "" {
 		opts = append(opts, credentialsOpt...)
 	} else {
+		if !args.Settings.Version.IsActive(ctx, clusterversion.SupportAssumeRoleAuth) {
+			return nil, errors.New("cannot authenticate to cloud storage via assume role until cluster has fully upgraded to 22.2")
+		}
+
 		assumeOpt, err := createImpersonateCredentials(ctx, conf.AssumeRole, conf.AssumeRoleDelegates, []string{scope}, credentialsOpt...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to assume role")
diff --git a/pkg/clusterversion/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go
@@ -298,6 +298,9 @@ const (
 	// leases to nodes that (i) don't expect them for certain keyspans, and (ii)
 	// don't know to upgrade them to efficient epoch-based ones.
 	EnableLeaseUpgrade
+	// SupportAssumeRoleAuth is the version where assume role authorization is
+	// supported in cloud storage and KMS.
+	SupportAssumeRoleAuth
 
 	// *************************************************
 	// Step (1): Add new versions here.
@@ -481,6 +484,10 @@ var rawVersionsSingleton = keyedVersions{
 		Key:     EnableLeaseUpgrade,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 72},
 	},
+	{
+		Key:     SupportAssumeRoleAuth,
+		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 74},
+	},
 
 	// *************************************************
 	// Step (2): Add new versions here.
diff --git a/pkg/clusterversion/key_string.go b/pkg/clusterversion/key_string.go
