feat(spanner): Add gRPC A66/A94 metrics (#13825)

HailongWen · web-flow · commit d695802a2981 · 2026-03-05T11:12:25.000+05:30
This change adds `grpc.client.attempt.started` metric in gRPC A66 and
all the
subchannel metrics in gRPC A94.

Similar to the Java counterpart, this change also updates the exporter
to
handle the gauge metric correctly. Local tests show confirm that
`Int64UpDownCounter` generates `Sum` data points (with
`IsMonotonic=false`)
instead of `Gauge` data.
diff --git a/spanner/metrics.go b/spanner/metrics.go
@@ -69,6 +69,9 @@ const (
 	metricLabelKeyGRPCLBPickResult      = "grpc.lb.pick_result"
 	metricLabelKeyGRPCLBDataPlaneTarget = "grpc.lb.rls.data_plane_target"
 	metricLabelKeyGRPCXDSResourceType   = "grpc.xds.resource_type"
+	metricLabelKeyGRPCLBLocality        = "grpc.lb.locality"
+	metricLabelKeyGRPCLBBackendService  = "grpc.lb.backend_service"
+	metricLabelKeyGRPCDisconnectError   = "grpc.disconnect_error"
 
 	// Metric names
 	metricNameOperationLatencies        = "operation_latencies"
@@ -222,12 +225,23 @@ var (
 	}
 
 	grpcMetricsToEnable = []string{
+		"grpc.client.attempt.started",
+		"grpc.subchannel.open_connections",
+		"grpc.subchannel.disconnections",
+		"grpc.subchannel.connection_attempts_succeeded",
+		"grpc.subchannel.connection_attempts_failed",
 		"grpc.lb.rls.default_target_picks",
 		"grpc.lb.rls.target_picks",
 		"grpc.xds_client.server_failure",
 		"grpc.xds_client.resource_updates_invalid",
 		"grpc.xds_client.resource_updates_valid",
 	}
+
+	grpcOptionalLabels = []string{
+		"grpc.disconnect_error",
+		"grpc.lb.backend_service",
+		"grpc.lb.locality",
+	}
 )
 
 type metricInfo struct {
@@ -299,8 +313,9 @@ func newBuiltinMetricsTracerFactory(ctx context.Context, dbpath, compression str
 
 		if isEnableGRPCBuiltInMetrics {
 			mo := opentelemetry.MetricsOptions{
-				MeterProvider: meterProvider,
-				Metrics:       stats.NewMetrics(grpcMetricsToEnable...),
+				MeterProvider:  meterProvider,
+				Metrics:        stats.NewMetrics(grpcMetricsToEnable...),
+				OptionalLabels: grpcOptionalLabels,
 			}
 
 			// Configure gRPC dial options to enable gRPC metrics collection and static method call option.
diff --git a/spanner/metrics_monitoring_exporter.go b/spanner/metrics_monitoring_exporter.go
@@ -67,6 +67,9 @@ var (
 		metricLabelKeyGRPCLBPickResult:      true,
 		metricLabelKeyGRPCLBDataPlaneTarget: true,
 		metricLabelKeyGRPCXDSResourceType:   true,
+		metricLabelKeyGRPCLBLocality:        true,
+		metricLabelKeyGRPCLBBackendService:  true,
+		metricLabelKeyGRPCDisconnectError:   true,
 		metricLabelKeyClientUID:             true,
 		metricLabelKeyClientName:            true,
 		metricLabelKeyDatabase:              true,
@@ -290,7 +293,10 @@ func (me *monitoringExporter) recordToTimeSeriesPb(m otelmetricdata.Metrics) ([]
 			metric, mr := me.recordToMetricAndMonitoredResourcePbs(m, point.Attributes)
 			var ts *monitoringpb.TimeSeries
 			var err error
-			ts, err = sumToTimeSeries[int64](point, m, mr)
+			// Int64UpDownCounter contains Sum data with IsMonotonic = false.
+			// See https://tinyurl.com/yzks9ene.
+			isGauge := !a.IsMonotonic
+			ts, err = sumToTimeSeries[int64](point, m, mr, isGauge)
 			if err != nil {
 				errs = append(errs, err)
 				continue
@@ -304,16 +310,20 @@ func (me *monitoringExporter) recordToTimeSeriesPb(m otelmetricdata.Metrics) ([]
 	return tss, errors.Join(errs...)
 }
 
-func sumToTimeSeries[N int64 | float64](point otelmetricdata.DataPoint[N], metrics otelmetricdata.Metrics, mr *monitoredrespb.MonitoredResource) (*monitoringpb.TimeSeries, error) {
-	interval, err := toNonemptyTimeIntervalpb(point.StartTime, point.Time)
+func sumToTimeSeries[N int64 | float64](point otelmetricdata.DataPoint[N], metrics otelmetricdata.Metrics, mr *monitoredrespb.MonitoredResource, isGauge bool) (*monitoringpb.TimeSeries, error) {
+	interval, err := toNonemptyTimeIntervalpb(point.StartTime, point.Time, isGauge)
 	if err != nil {
 		return nil, err
 	}
 	value, valueType := numberDataPointToValue[N](point)
+	metricKind := googlemetricpb.MetricDescriptor_CUMULATIVE
+	if isGauge {
+		metricKind = googlemetricpb.MetricDescriptor_GAUGE
+	}
 	return &monitoringpb.TimeSeries{
 		Resource:   mr,
 		Unit:       string(metrics.Unit),
-		MetricKind: googlemetricpb.MetricDescriptor_CUMULATIVE,
+		MetricKind: metricKind,
 		ValueType:  valueType,
 		Points: []*monitoringpb.Point{{
 			Interval: interval,
@@ -323,7 +333,7 @@ func sumToTimeSeries[N int64 | float64](point otelmetricdata.DataPoint[N], metri
 }
 
 func histogramToTimeSeries[N int64 | float64](point otelmetricdata.HistogramDataPoint[N], metrics otelmetricdata.Metrics, mr *monitoredrespb.MonitoredResource) (*monitoringpb.TimeSeries, error) {
-	interval, err := toNonemptyTimeIntervalpb(point.StartTime, point.Time)
+	interval, err := toNonemptyTimeIntervalpb(point.StartTime, point.Time, false)
 	if err != nil {
 		return nil, err
 	}
@@ -344,11 +354,13 @@ func histogramToTimeSeries[N int64 | float64](point otelmetricdata.HistogramData
 	}, nil
 }
 
-func toNonemptyTimeIntervalpb(start, end time.Time) (*monitoringpb.TimeInterval, error) {
+func toNonemptyTimeIntervalpb(start, end time.Time, isGauge bool) (*monitoringpb.TimeInterval, error) {
 	// The end time of a new interval must be at least a millisecond after the end time of the
 	// previous interval, for all non-gauge types.
 	// https://cloud.google.com/monitoring/api/ref_v3/rpc/google.monitoring.v3#timeinterval
-	if end.Sub(start).Milliseconds() <= 1 {
+	if isGauge {
+		end = start
+	} else if end.Sub(start).Milliseconds() <= 1 {
 		end = start.Add(time.Millisecond)
 	}
 	startpb := timestamppb.New(start)
