feat(storage): support Bucket custom placement config (#9481)

coryan · web-flow · commit b529b8c32164 · 2022-07-14T17:23:58.000-04:00
diff --git a/google/cloud/storage/bucket_metadata.cc b/google/cloud/storage/bucket_metadata.cc
@@ -133,6 +133,12 @@ std::ostream& operator<<(std::ostream& os, BucketRetentionPolicy const& rhs) {
             << ", locked=" << rhs.is_locked << "}";
 }
 
+std::ostream& operator<<(std::ostream& os,
+                         BucketCustomPlacementConfig const& rhs) {
+  return os << "BucketCustomPlacementConfig=["
+            << absl::StrJoin(rhs.data_locations, ", ") << "]";
+}
+
 bool operator==(BucketMetadata const& lhs, BucketMetadata const& rhs) {
   return static_cast<internal::CommonMetadata<BucketMetadata> const&>(lhs) ==
              rhs &&
@@ -148,7 +154,8 @@ bool operator==(BucketMetadata const& lhs, BucketMetadata const& rhs) {
          lhs.logging_ == rhs.logging_ && lhs.labels_ == rhs.labels_ &&
          lhs.retention_policy_ == rhs.retention_policy_ &&
          lhs.rpo_ == rhs.rpo_ && lhs.versioning_ == rhs.versioning_ &&
-         lhs.website_ == rhs.website_;
+         lhs.website_ == rhs.website_ &&
+         lhs.custom_placement_config_ == rhs.custom_placement_config_;
 }
 
 std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs) {
@@ -245,6 +252,12 @@ std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs) {
        << ", website.not_found_page=" << rhs.website().not_found_page;
   }
 
+  if (rhs.has_custom_placement_config()) {
+    os << ", custom_placement_config.data_locations=["
+       << absl::StrJoin(rhs.custom_placement_config().data_locations, ", ")
+       << "]";
+  }
+
   return os << "}";
 }
 
diff --git a/google/cloud/storage/bucket_metadata.h b/google/cloud/storage/bucket_metadata.h
@@ -521,6 +521,56 @@ inline bool operator>=(BucketWebsite const& lhs, BucketWebsite const& rhs) {
   return std::rel_ops::operator>=(lhs, rhs);
 }
 
+/**
+ * Configuration for Custom Dual Regions.
+ *
+ * It should specify precisely two eligible regions within the same Multiregion.
+ *
+ * @see Additional information on custom dual regions in the
+ *     [feature documentation][cdr-link].
+ *
+ * [cdr-link]: https://cloud.google.com/storage/docs/locations
+ */
+struct BucketCustomPlacementConfig {
+  std::vector<std::string> data_locations;
+};
+
+//@{
+/// @name Comparison operators for BucketCustomPlacementConfig.
+inline bool operator==(BucketCustomPlacementConfig const& lhs,
+                       BucketCustomPlacementConfig const& rhs) {
+  return lhs.data_locations == rhs.data_locations;
+}
+
+inline bool operator<(BucketCustomPlacementConfig const& lhs,
+                      BucketCustomPlacementConfig const& rhs) {
+  return lhs.data_locations < rhs.data_locations;
+}
+
+inline bool operator!=(BucketCustomPlacementConfig const& lhs,
+                       BucketCustomPlacementConfig const& rhs) {
+  return std::rel_ops::operator!=(lhs, rhs);
+}
+
+inline bool operator>(BucketCustomPlacementConfig const& lhs,
+                      BucketCustomPlacementConfig const& rhs) {
+  return std::rel_ops::operator>(lhs, rhs);
+}
+
+inline bool operator<=(BucketCustomPlacementConfig const& lhs,
+                       BucketCustomPlacementConfig const& rhs) {
+  return std::rel_ops::operator<=(lhs, rhs);
+}
+
+inline bool operator>=(BucketCustomPlacementConfig const& lhs,
+                       BucketCustomPlacementConfig const& rhs) {
+  return std::rel_ops::operator>=(lhs, rhs);
+}
+//@}
+
+std::ostream& operator<<(std::ostream& os,
+                         BucketCustomPlacementConfig const& rhs);
+
 /**
  * Represents a Google Cloud Storage Bucket Metadata object.
  */
@@ -912,6 +962,30 @@ class BucketMetadata : private internal::CommonMetadata<BucketMetadata> {
   }
   ///@}
 
+  /// @name Accessors and modifiers for custom placement configuration.
+  ///@{
+  bool has_custom_placement_config() const {
+    return custom_placement_config_.has_value();
+  }
+  BucketCustomPlacementConfig const& custom_placement_config() const {
+    return *custom_placement_config_;
+  }
+  absl::optional<BucketCustomPlacementConfig> const&
+  custom_placement_config_as_optional() const {
+    return custom_placement_config_;
+  }
+  /// Placement configuration can only be set when the bucket is created.
+  BucketMetadata& set_custom_placement_config(BucketCustomPlacementConfig v) {
+    custom_placement_config_ = std::move(v);
+    return *this;
+  }
+  /// Placement configuration can only be set when the bucket is created.
+  BucketMetadata& reset_custom_placement_config() {
+    custom_placement_config_.reset();
+    return *this;
+  }
+  ///@}
+
   friend bool operator==(BucketMetadata const& lhs, BucketMetadata const& rhs);
   friend bool operator!=(BucketMetadata const& lhs, BucketMetadata const& rhs) {
     return !(lhs == rhs);
@@ -940,6 +1014,7 @@ class BucketMetadata : private internal::CommonMetadata<BucketMetadata> {
   std::string rpo_;
   absl::optional<BucketVersioning> versioning_;
   absl::optional<BucketWebsite> website_;
+  absl::optional<BucketCustomPlacementConfig> custom_placement_config_;
 };
 
 std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs);
diff --git a/google/cloud/storage/bucket_metadata_test.cc b/google/cloud/storage/bucket_metadata_test.cc
@@ -163,6 +163,9 @@ BucketMetadata CreateBucketMetadataForTest() {
       "website": {
         "mainPageSuffix": "index.html",
         "notFoundPage": "404.html"
+      },
+      "customPlacementConfig": {
+        "dataLocations": ["us-central1", "us-east1"]
       }
 })""";
   return internal::BucketMetadataParser::FromString(text).value();
@@ -277,6 +280,14 @@ TEST(BucketMetadataTest, Parse) {
   ASSERT_TRUE(actual.has_website());
   EXPECT_EQ("index.html", actual.website().main_page_suffix);
   EXPECT_EQ("404.html", actual.website().not_found_page);
+
+  // custom placement config
+  ASSERT_TRUE(actual.has_custom_placement_config());
+  EXPECT_THAT(actual.custom_placement_config().data_locations,
+              ElementsAre("us-central1", "us-east1"));
+  ASSERT_TRUE(actual.custom_placement_config_as_optional().has_value());
+  EXPECT_THAT(actual.custom_placement_config_as_optional()->data_locations,
+              ElementsAre("us-central1", "us-east1"));
 }
 
 /// @test Verify that the IOStream operator works as expected.
@@ -350,6 +361,12 @@ TEST(BucketMetadataTest, IOStream) {
   // website()
   EXPECT_THAT(actual, HasSubstr("index.html"));
   EXPECT_THAT(actual, HasSubstr("404.html"));
+
+  // custom_placement_config()
+  EXPECT_THAT(
+      actual,
+      HasSubstr(
+          "custom_placement_config.data_locations=[us-central1, us-east1]"));
 }
 
 /// @test Verify we can convert a BucketMetadata object to a JSON string.
@@ -476,6 +493,13 @@ TEST(BucketMetadataTest, ToJsonString) {
   ASSERT_TRUE(actual["website"].is_object()) << actual;
   EXPECT_EQ("index.html", actual["website"].value("mainPageSuffix", ""));
   EXPECT_EQ("404.html", actual["website"].value("notFoundPage", ""));
+
+  // custom_placement_config()
+  ASSERT_TRUE(actual.contains("customPlacementConfig")) << actual;
+  auto expected_custom_placement_config = nlohmann::json{
+      {"dataLocations", std::vector<std::string>{"us-central1", "us-east1"}},
+  };
+  EXPECT_EQ(actual["customPlacementConfig"], expected_custom_placement_config);
 }
 
 TEST(BucketMetadataTest, ToJsonLifecycleRoundtrip) {
@@ -863,6 +887,39 @@ TEST(BucketMetadataTest, ResetWebsite) {
   EXPECT_THAT(os.str(), Not(HasSubstr("website.")));
 }
 
+/// @test Verify we can set the custom_placement_config field in BucketMetadata.
+TEST(BucketMetadataTest, SetCustomPlacementConfig) {
+  auto expected = CreateBucketMetadataForTest();
+  auto copy = expected;
+  copy.set_custom_placement_config(
+      BucketCustomPlacementConfig{{"test-location-1", "test-location-2"}});
+  ASSERT_TRUE(copy.has_custom_placement_config());
+  EXPECT_THAT(copy.custom_placement_config().data_locations,
+              ElementsAre("test-location-1", "test-location-2"));
+  ASSERT_TRUE(copy.custom_placement_config_as_optional().has_value());
+  EXPECT_THAT(copy.custom_placement_config_as_optional()->data_locations,
+              ElementsAre("test-location-1", "test-location-2"));
+  EXPECT_NE(copy, expected);
+  std::ostringstream os;
+  os << copy;
+  EXPECT_THAT(os.str(), HasSubstr("custom_placement_config"));
+}
+
+/// @test Verify we can set the custom_placement_config field in BucketMetadata.
+TEST(BucketMetadataTest, ResetCustomPlacementConfig) {
+  auto expected = CreateBucketMetadataForTest();
+  EXPECT_TRUE(expected.has_custom_placement_config());
+  EXPECT_TRUE(expected.custom_placement_config_as_optional().has_value());
+  auto copy = expected;
+  copy.reset_custom_placement_config();
+  EXPECT_FALSE(copy.has_custom_placement_config());
+  EXPECT_FALSE(copy.custom_placement_config_as_optional().has_value());
+  EXPECT_NE(copy, expected);
+  std::ostringstream os;
+  os << copy;
+  EXPECT_THAT(os.str(), Not(HasSubstr("custom_placement_config")));
+}
+
 TEST(BucketMetadataPatchBuilder, SetAcl) {
   BucketMetadataPatchBuilder builder;
   builder.SetAcl({internal::BucketAccessControlParser::FromString(
diff --git a/google/cloud/storage/examples/storage_bucket_samples.cc b/google/cloud/storage/examples/storage_bucket_samples.cc
@@ -146,7 +146,8 @@ void CreateBucketDualRegion(google::cloud::storage::Client client,
      std::string const& region_a, std::string const& region_b) {
     auto metadata = client.CreateBucket(
         bucket_name,
-        gcs::BucketMetadata().set_location(region_a + '+' + region_b));
+        gcs::BucketMetadata().set_custom_placement_config(
+            gcs::BucketCustomPlacementConfig{{region_a, region_b}}));
     if (!metadata) throw std::runtime_error(metadata.status().message());
 
     std::cout << "Bucket " << metadata->name() << " created."
@@ -711,7 +712,7 @@ void RunAll(std::vector<std::string> const& argv) {
   DeleteBucket(client, {bucket_name});
 
   std::cout << "\nRunning CreateBucketDualRegion example" << std::endl;
-  CreateBucketDualRegion(client, {dual_bucket_name, "us-east1", "us-central1"});
+  CreateBucketDualRegion(client, {dual_bucket_name, "us-east4", "us-central1"});
 
   (void)client.DeleteBucket(dual_bucket_name);
 
diff --git a/google/cloud/storage/internal/bucket_metadata_parser.cc b/google/cloud/storage/internal/bucket_metadata_parser.cc
@@ -235,6 +235,29 @@ Status ParseWebsite(absl::optional<BucketWebsite>& website,
   return Status{};
 }
 
+Status ParseCustomPlacementConfig(
+    absl::optional<BucketCustomPlacementConfig>& lhs,
+    nlohmann::json const& json) {
+  if (!json.contains("customPlacementConfig")) return Status{};
+  auto const& field = json["customPlacementConfig"];
+  auto error = [] {
+    return Status{StatusCode::kInvalidArgument,
+                  "malformed customPlacementConfig"};
+  };
+  if (!field.is_object()) return error();
+  if (!field.contains("dataLocations")) return Status{};
+  auto const& locations = field["dataLocations"];
+  if (!locations.is_array()) return error();
+
+  BucketCustomPlacementConfig value;
+  for (auto const& i : locations.items()) {
+    if (!i.value().is_string()) return error();
+    value.data_locations.push_back(i.value().get<std::string>());
+  }
+  lhs = std::move(value);
+  return Status{};
+}
+
 void ToJsonAcl(nlohmann::json& json, BucketMetadata const& meta) {
   if (meta.acl().empty()) return;
   nlohmann::json value;
@@ -416,6 +439,14 @@ void ToJsonWebsite(nlohmann::json& json, BucketMetadata const& meta) {
   json["website"] = std::move(value);
 }
 
+void ToJsonCustomPlacementConfig(nlohmann::json& json,
+                                 BucketMetadata const& meta) {
+  if (!meta.has_custom_placement_config()) return;
+  json["customPlacementConfig"] = nlohmann::json{
+      {"dataLocations", meta.custom_placement_config().data_locations},
+  };
+}
+
 }  // namespace
 
 StatusOr<BucketMetadata> BucketMetadataParser::FromJson(
@@ -484,6 +515,9 @@ StatusOr<BucketMetadata> BucketMetadataParser::FromJson(
       [](BucketMetadata& meta, nlohmann::json const& json) {
         return ParseWebsite(meta.website_, json);
       },
+      [](BucketMetadata& meta, nlohmann::json const& json) {
+        return ParseCustomPlacementConfig(meta.custom_placement_config_, json);
+      },
   };
 
   BucketMetadata meta{};
@@ -525,6 +559,7 @@ std::string BucketMetadataToJsonString(BucketMetadata const& meta) {
   ToJsonStorageClass(json, meta);
   ToJsonVersioning(json, meta);
   ToJsonWebsite(json, meta);
+  ToJsonCustomPlacementConfig(json, meta);
 
   return json.dump();
 }
diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc b/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc
@@ -100,6 +100,10 @@ google::storage::v2::Bucket GrpcBucketMetadataParser::ToProto(
   if (rhs.has_iam_configuration()) {
     *result.mutable_iam_config() = ToProto(rhs.iam_configuration());
   }
+  if (rhs.has_custom_placement_config()) {
+    *result.mutable_custom_placement_config() =
+        ToProto(rhs.custom_placement_config());
+  }
   return result;
 }
 
@@ -169,6 +173,10 @@ BucketMetadata GrpcBucketMetadataParser::FromProto(
   }
   if (rhs.has_versioning()) metadata.versioning_ = FromProto(rhs.versioning());
   if (rhs.has_website()) metadata.website_ = FromProto(rhs.website());
+  if (rhs.has_custom_placement_config()) {
+    metadata.custom_placement_config_ =
+        FromProto(rhs.custom_placement_config());
+  }
 
   return metadata;
 }
@@ -479,6 +487,24 @@ BucketWebsite GrpcBucketMetadataParser::FromProto(
   return result;
 }
 
+google::storage::v2::Bucket::CustomPlacementConfig
+GrpcBucketMetadataParser::ToProto(BucketCustomPlacementConfig rhs) {
+  google::storage::v2::Bucket::CustomPlacementConfig result;
+  for (auto& l : rhs.data_locations) {
+    *result.add_data_locations() = std::move(l);
+  }
+  return result;
+}
+
+BucketCustomPlacementConfig GrpcBucketMetadataParser::FromProto(
+    google::storage::v2::Bucket::CustomPlacementConfig rhs) {
+  BucketCustomPlacementConfig result;
+  for (auto& l : *rhs.mutable_data_locations()) {
+    result.data_locations.push_back(std::move(l));
+  }
+  return result;
+}
+
 }  // namespace internal
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace storage
diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser.h b/google/cloud/storage/internal/grpc_bucket_metadata_parser.h
@@ -80,6 +80,11 @@ struct GrpcBucketMetadataParser {
 
   static google::storage::v2::Bucket::Website ToProto(BucketWebsite rhs);
   static BucketWebsite FromProto(google::storage::v2::Bucket::Website rhs);
+
+  static google::storage::v2::Bucket::CustomPlacementConfig ToProto(
+      BucketCustomPlacementConfig rhs);
+  static BucketCustomPlacementConfig FromProto(
+      google::storage::v2::Bucket::CustomPlacementConfig rhs);
 };
 
 }  // namespace internal
diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc b/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc
