feat: support min_num_instances for primary worker and InstanceFlexibilityPolicy for secondary worker

Google APIs · copybara-github · commit 4a5a6fd4522d · 2023-08-22T09:31:19.000-07:00
PiperOrigin-RevId: 559135594
diff --git a/google/cloud/dataproc/v1/BUILD.bazel b/google/cloud/dataproc/v1/BUILD.bazel
@@ -1,4 +1,13 @@
 # This file was automatically generated by BuildFileGenerator
+# https://github.com/googleapis/rules_gapic/tree/master/bazel
+
+# Most of the manual changes to this file will be overwritten.
+# It's **only** allowed to change the following rule attribute values:
+# - names of *_gapic_assembly_* rules
+# - certain parameters of *_gapic_library rules, including but not limited to:
+#    * extra_protoc_parameters
+#    * extra_protoc_file_parameters
+# The complete list of preserved parameters can be found in the source code.
 
 # This is an API workspace, having public visibility by default makes perfect sense.
 package(default_visibility = ["//visibility:public"])
@@ -39,6 +48,7 @@ proto_library_with_info(
     deps = [
         ":dataproc_proto",
         "//google/cloud:common_resources_proto",
+        "//google/iam/v1:iam_policy_proto",
     ],
 )
 
@@ -73,29 +83,32 @@ java_gapic_library(
     rest_numeric_enums = True,
     service_yaml = "dataproc_v1.yaml",
     test_deps = [
+        "//google/iam/v1:iam_java_grpc",
         ":dataproc_java_grpc",
     ],
     transport = "grpc+rest",
     deps = [
         ":dataproc_java_proto",
         "//google/api:api_java_proto",
+        "//google/iam/v1:iam_java_proto",
     ],
 )
 
 java_gapic_test(
     name = "dataproc_java_gapic_test_suite",
     test_classes = [
-        # Add AutoscalingPolicyService to gapic yaml
         "com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClientHttpJsonTest",
         "com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClientTest",
+        "com.google.cloud.dataproc.v1.BatchControllerClientHttpJsonTest",
+        "com.google.cloud.dataproc.v1.BatchControllerClientTest",
         "com.google.cloud.dataproc.v1.ClusterControllerClientHttpJsonTest",
         "com.google.cloud.dataproc.v1.ClusterControllerClientTest",
         "com.google.cloud.dataproc.v1.JobControllerClientHttpJsonTest",
         "com.google.cloud.dataproc.v1.JobControllerClientTest",
-        "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientHttpJsonTest",
-        "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientTest",
         "com.google.cloud.dataproc.v1.NodeGroupControllerClientHttpJsonTest",
         "com.google.cloud.dataproc.v1.NodeGroupControllerClientTest",
+        "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientHttpJsonTest",
+        "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientTest",
     ],
     runtime_deps = [":dataproc_java_gapic_test"],
 )
@@ -146,7 +159,6 @@ go_gapic_library(
     transport = "grpc+rest",
     deps = [
         ":dataproc_go_proto",
-        "//google/cloud/location:location_go_proto",
         "//google/iam/v1:iam_go_proto",
         "//google/longrunning:longrunning_go_proto",
         "@com_google_cloud_go_longrunning//:go_default_library",
@@ -160,6 +172,7 @@ go_gapic_assembly_pkg(
     name = "gapi-cloud-dataproc-v1-go",
     deps = [
         ":dataproc_go_gapic",
+        ":dataproc_go_gapic_srcjar-metadata.srcjar",
         ":dataproc_go_gapic_srcjar-snippets.srcjar",
         ":dataproc_go_gapic_srcjar-test.srcjar",
         ":dataproc_go_proto",
@@ -185,7 +198,6 @@ py_gapic_library(
     transport = "grpc+rest",
     deps = [
         "//google/iam/v1:iam_policy_py_proto",
-        "//google/iam/v1:policy_py_proto",
     ],
 )
 
@@ -199,6 +211,7 @@ py_test(
     deps = [":dataproc_py_gapic"],
 )
 
+# Open Source Packages
 py_gapic_assembly_pkg(
     name = "dataproc-v1-py",
     deps = [
@@ -224,13 +237,14 @@ php_proto_library(
 php_gapic_library(
     name = "dataproc_php_gapic",
     srcs = [":dataproc_proto_with_info"],
-    gapic_yaml = "dataproc_gapic.yaml",
     grpc_service_config = "dataproc_grpc_service_config.json",
     migration_mode = "MIGRATION_MODE_UNSPECIFIED",
     rest_numeric_enums = True,
     service_yaml = "dataproc_v1.yaml",
     transport = "grpc+rest",
-    deps = [":dataproc_php_proto"],
+    deps = [
+        ":dataproc_php_proto",
+    ],
 )
 
 # Open Source Packages
@@ -299,11 +313,11 @@ ruby_cloud_gapic_library(
     name = "dataproc_ruby_gapic",
     srcs = [":dataproc_proto_with_info"],
     extra_protoc_parameters = [
-        "ruby-cloud-gem-name=google-cloud-dataproc-v1",
-        "ruby-cloud-env-prefix=DATAPROC",
-        "ruby-cloud-product-url=https://cloud.google.com/dataproc",
         "ruby-cloud-api-id=dataproc.googleapis.com",
         "ruby-cloud-api-shortname=dataproc",
+        "ruby-cloud-env-prefix=DATAPROC",
+        "ruby-cloud-gem-name=google-cloud-dataproc-v1",
+        "ruby-cloud-product-url=https://cloud.google.com/dataproc",
     ],
     grpc_service_config = "dataproc_grpc_service_config.json",
     rest_numeric_enums = True,
@@ -340,6 +354,7 @@ load(
 
 csharp_proto_library(
     name = "dataproc_csharp_proto",
+    extra_opts = [],
     deps = [":dataproc_proto"],
 )
 
diff --git a/google/cloud/dataproc/v1/clusters.proto b/google/cloud/dataproc/v1/clusters.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -489,10 +489,10 @@ message GceClusterConfig {
   // instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
   repeated string tags = 4;
 
-  // The Compute Engine metadata entries to add to all instances (see
+  // Optional. The Compute Engine metadata entries to add to all instances (see
   // [Project and instance
   // metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
-  map<string, string> metadata = 5;
+  map<string, string> metadata = 5 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. Reservation Affinity for consuming Zonal reservation.
   ReservationAffinity reservation_affinity = 11
@@ -600,6 +600,10 @@ message InstanceGroupConfig {
   repeated string instance_names = 2
       [(google.api.field_behavior) = OUTPUT_ONLY];
 
+  // Output only. List of references to Compute Engine instances.
+  repeated InstanceReference instance_references = 11
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
   // Optional. The Compute Engine image resource used for cluster instances.
   //
   // The URI can represent an image or image family.
@@ -666,6 +670,44 @@ message InstanceGroupConfig {
   // See [Dataproc -> Minimum CPU
   // Platform](https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu).
   string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. The minimum number of instances to create.
+  // If min_num_instances is set, min_num_instances is used for a criteria to
+  // decide the cluster. Cluster creation will be failed by being an error state
+  // if the total number of instances created is less than the
+  // min_num_instances.
+  // For example, given that num_instances = 5 and min_num_instances = 3,
+  // * if 4 instances are created and then registered successfully but one
+  // instance is failed, the failed VM will be deleted and the cluster will be
+  // resized to 4 instances in running state.
+  // * if 2 instances are created successfully and 3 instances are failed,
+  // the cluster will be in an error state and does not delete failed VMs for
+  // debugging.
+  // * if 2 instance are created and then registered successfully but 3
+  // instances are failed to initialize, the cluster will be in an error state
+  // and does not delete failed VMs for debugging.
+  // NB: This can only be set for primary workers now.
+  int32 min_num_instances = 12 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Instance flexibility Policy allowing a mixture of VM shapes and
+  // provisioning models.
+  InstanceFlexibilityPolicy instance_flexibility_policy = 13
+      [(google.api.field_behavior) = OPTIONAL];
+}
+
+// A reference to a Compute Engine instance.
+message InstanceReference {
+  // The user-friendly name of the Compute Engine instance.
+  string instance_name = 1;
+
+  // The unique identifier of the Compute Engine instance.
+  string instance_id = 2;
+
+  // The public RSA key used for sharing data with this instance.
+  string public_key = 3;
+
+  // The public ECIES key used for sharing data with this instance.
+  string public_ecies_key = 4;
 }
 
 // Specifies the resources used to actively manage an instance group.
@@ -677,6 +719,48 @@ message ManagedGroupConfig {
   // Output only. The name of the Instance Group Manager for this group.
   string instance_group_manager_name = 2
       [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The partial URI to the instance group manager for this group.
+  // E.g. projects/my-project/regions/us-central1/instanceGroupManagers/my-igm.
+  string instance_group_manager_uri = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Instance flexibility Policy allowing a mixture of VM shapes and provisioning
+// models.
+message InstanceFlexibilityPolicy {
+  // Defines machines types and a rank to which the machines types belong.
+  message InstanceSelection {
+    // Optional. Full machine-type names, e.g. "n1-standard-16".
+    repeated string machine_types = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Preference of this instance selection. Lower number means
+    // higher preference. Dataproc will first try to create a VM based on the
+    // machine-type with priority rank and fallback to next rank based on
+    // availability. Machine types and instance selections with the same
+    // priority have the same preference.
+    int32 rank = 2 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Defines a mapping from machine types to the number of VMs that are created
+  // with each machine type.
+  message InstanceSelectionResult {
+    // Output only. Full machine-type names, e.g. "n1-standard-16".
+    optional string machine_type = 1
+        [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Output only. Number of VM provisioned with the machine_type.
+    optional int32 vm_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  }
+
+  // Optional. List of instance selection options that the group will use when
+  // creating new VMs.
+  repeated InstanceSelection instance_selection_list = 2
+      [(google.api.field_behavior) = OPTIONAL];
+
+  // Output only. A list of instance selection results in the group.
+  repeated InstanceSelectionResult instance_selection_results = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // Specifies the type and number of accelerator cards attached to the instances
@@ -844,6 +928,9 @@ message ClusterStatus {
 
     // The cluster is being started. It is not ready for use.
     STARTING = 8;
+
+    // The cluster is being repaired. It is not ready for use.
+    REPAIRING = 10;
   }
 
   // The cluster substate.
@@ -1075,16 +1162,16 @@ message ClusterMetrics {
 
 // Dataproc metric config.
 message DataprocMetricConfig {
-  // A source for the collection of Dataproc OSS metrics (see [available OSS
+  // A source for the collection of Dataproc custom metrics (see [Custom
   // metrics]
-  // (https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)).
+  // (https://cloud.google.com//dataproc/docs/guides/dataproc-metrics#custom_metrics)).
   enum MetricSource {
     // Required unspecified metric source.
     METRIC_SOURCE_UNSPECIFIED = 0;
 
-    // Default monitoring agent metrics. If this source is enabled,
+    // Monitoring agent metrics. If this source is enabled,
     // Dataproc enables the monitoring agent in Compute Engine,
-    // and collects default monitoring agent metrics, which are published
+    // and collects monitoring agent metrics, which are published
     // with an `agent.googleapis.com` prefix.
     MONITORING_AGENT_DEFAULTS = 1;
 
@@ -1107,17 +1194,17 @@ message DataprocMetricConfig {
     HIVEMETASTORE = 7;
   }
 
-  // A Dataproc OSS metric.
+  // A Dataproc custom metric.
   message Metric {
-    // Required. Default metrics are collected unless `metricOverrides` are
-    // specified for the metric source (see [Available OSS metrics]
-    // (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics)
+    // Required. A standard set of metrics is collected unless `metricOverrides`
+    // are specified for the metric source (see [Custom metrics]
+    // (https://cloud.google.com/dataproc/docs/guides/dataproc-metrics#custom_metrics)
     // for more information).
     MetricSource metric_source = 1 [(google.api.field_behavior) = REQUIRED];
 
-    // Optional. Specify one or more [available OSS metrics]
-    // (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics)
-    // to collect for the metric course (for the `SPARK` metric source, any
+    // Optional. Specify one or more [Custom metrics]
+    // (https://cloud.google.com/dataproc/docs/guides/dataproc-metrics#custom_metrics)
+    // to collect for the metric course (for the `SPARK` metric source (any
     // [Spark metric]
     // (https://spark.apache.org/docs/latest/monitoring.html#metrics) can be
     // specified).
@@ -1137,13 +1224,13 @@ message DataprocMetricConfig {
     //
     // Notes:
     //
-    // * Only the specified overridden metrics will be collected for the
+    // * Only the specified overridden metrics are collected for the
     //   metric source. For example, if one or more `spark:executive` metrics
-    //   are listed as metric overrides, other `SPARK` metrics will not be
-    //   collected. The collection of the default metrics for other OSS metric
-    //   sources is unaffected. For example, if both `SPARK` andd `YARN` metric
-    //   sources are enabled, and overrides are provided for Spark metrics only,
-    //   all default YARN metrics will be collected.
+    //   are listed as metric overrides, other `SPARK` metrics are not
+    //   collected. The collection of the metrics for other enabled custom
+    //   metric sources is unaffected. For example, if both `SPARK` andd `YARN`
+    //   metric sources are enabled, and overrides are provided for Spark
+    //   metrics only, all YARN metrics are collected.
     repeated string metric_overrides = 2
         [(google.api.field_behavior) = OPTIONAL];
   }
diff --git a/google/cloud/dataproc/v1/node_groups.proto b/google/cloud/dataproc/v1/node_groups.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -144,7 +144,7 @@ message ResizeNodeGroupRequest {
   // underscores (_), and hyphens (-). The maximum length is 40 characters.
   string request_id = 3 [(google.api.field_behavior) = OPTIONAL];
 
-  // Optional. Timeout for graceful YARN decomissioning. [Graceful
+  // Optional. Timeout for graceful YARN decommissioning. [Graceful
   // decommissioning]
   // (https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/scaling-clusters#graceful_decommissioning)
   // allows the removal of nodes from the Compute Engine node group
