Skip to content

Commit 04bd623

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: Support picosecond timestamp precision in BigQuery Storage API
docs: A comment for method `ReadRows` in service `BigQueryRead` is changed docs: A comment for message `AppendRowsRequest` is changed docs: A comment for field `writer_schema` in message `.google.cloud.bigquery.storage.v1.AppendRowsRequest` is changed docs: A comment for field `rows` in message `.google.cloud.bigquery.storage.v1.AppendRowsRequest` is changed docs: A comment for field `arrow_rows` in message `.google.cloud.bigquery.storage.v1.AppendRowsRequest` is changed docs: A comment for field `default_missing_value_interpretation` in message `.google.cloud.bigquery.storage.v1.AppendRowsRequest` is changed docs: A comment for field `location` in message `.google.cloud.bigquery.storage.v1.WriteStream` is changed PiperOrigin-RevId: 829486853
1 parent 2223580 commit 04bd623

File tree

7 files changed

+102
-27
lines changed

7 files changed

+102
-27
lines changed

‎google/cloud/bigquery/storage/v1/arrow.proto‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,32 @@ message ArrowSerializationOptions {
5858
ZSTD = 2;
5959
}
6060

61+
// The precision of the timestamp value in the Avro message. This precision
62+
// will **only** be applied to the column(s) with the `TIMESTAMP_PICOS` type.
63+
enum PicosTimestampPrecision {
64+
// Unspecified timestamp precision. The default precision is microseconds.
65+
PICOS_TIMESTAMP_PRECISION_UNSPECIFIED = 0;
66+
67+
// Timestamp values returned by Read API will be truncated to microsecond
68+
// level precision. The value will be encoded as Arrow TIMESTAMP type in a
69+
// 64 bit integer.
70+
TIMESTAMP_PRECISION_MICROS = 1;
71+
72+
// Timestamp values returned by Read API will be truncated to nanosecond
73+
// level precision. The value will be encoded as Arrow TIMESTAMP type in a
74+
// 64 bit integer.
75+
TIMESTAMP_PRECISION_NANOS = 2;
76+
77+
// Read API will return full precision picosecond value. The value will be
78+
// encoded as a string which conforms to ISO 8601 format.
79+
TIMESTAMP_PRECISION_PICOS = 3;
80+
}
81+
6182
// The compression codec to use for Arrow buffers in serialized record
6283
// batches.
6384
CompressionCodec buffer_compression = 2;
85+
86+
// Optional. Set timestamp precision option. If not set, the default precision
87+
// is microseconds.
88+
PicosTimestampPrecision picos_timestamp_precision = 3;
6489
}

‎google/cloud/bigquery/storage/v1/avro.proto‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,27 @@ message AvroRows {
4242

4343
// Contains options specific to Avro Serialization.
4444
message AvroSerializationOptions {
45+
// The precision of the timestamp value in the Avro message. This precision
46+
// will **only** be applied to the column(s) with the `TIMESTAMP_PICOS` type.
47+
enum PicosTimestampPrecision {
48+
// Unspecified timestamp precision. The default precision is microseconds.
49+
PICOS_TIMESTAMP_PRECISION_UNSPECIFIED = 0;
50+
51+
// Timestamp values returned by Read API will be truncated to microsecond
52+
// level precision. The value will be encoded as Avro TIMESTAMP type in a
53+
// 64 bit integer.
54+
TIMESTAMP_PRECISION_MICROS = 1;
55+
56+
// Timestamp values returned by Read API will be truncated to nanosecond
57+
// level precision. The value will be encoded as Avro TIMESTAMP type in a
58+
// 64 bit integer.
59+
TIMESTAMP_PRECISION_NANOS = 2;
60+
61+
// Read API will return full precision picosecond value. The value will be
62+
// encoded as a string which conforms to ISO 8601 format.
63+
TIMESTAMP_PRECISION_PICOS = 3;
64+
}
65+
4566
// Enable displayName attribute in Avro schema.
4667
//
4768
// The Avro specification requires field names to be alphanumeric. By
@@ -53,4 +74,8 @@ message AvroSerializationOptions {
5374
// value and populates a "displayName" attribute for every avro field with the
5475
// original column name.
5576
bool enable_display_name_attribute = 1;
77+
78+
// Optional. Set timestamp precision option. If not set, the default precision
79+
// is microseconds.
80+
PicosTimestampPrecision picos_timestamp_precision = 2;
5681
}

‎google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,25 @@
118118
"RESOURCE_EXHAUSTED"
119119
]
120120
}
121+
},
122+
{
123+
"name": [
124+
{
125+
"service": "google.cloud.bigquery.storage.v1.StreamMetastorePartitions",
126+
"method": "StreamMetastorePartitions"
127+
}
128+
],
129+
"timeout": "1200s",
130+
"retryPolicy": {
131+
"initialBackoff": "0.100s",
132+
"maxBackoff": "60s",
133+
"backoffMultiplier": 1.3,
134+
"retryableStatusCodes": [
135+
"DEADLINE_EXCEEDED",
136+
"UNAVAILABLE",
137+
"RESOURCE_EXHAUSTED"
138+
]
139+
}
121140
}
122141
]
123142
}

‎google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,6 @@ apis:
77
- name: google.cloud.bigquery.storage.v1.BigQueryRead
88
- name: google.cloud.bigquery.storage.v1.BigQueryWrite
99

10-
backend:
11-
rules:
12-
- selector: google.cloud.bigquery.storage.v1.BigQueryRead.CreateReadSession
13-
deadline: 120.0
14-
- selector: google.cloud.bigquery.storage.v1.BigQueryRead.ReadRows
15-
deadline: 21600.0
16-
- selector: google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream
17-
deadline: 120.0
18-
- selector: 'google.cloud.bigquery.storage.v1.BigQueryWrite.*'
19-
deadline: 120.0
20-
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStream
21-
deadline: 600.0
22-
2310
authentication:
2411
rules:
2512
- selector: 'google.cloud.bigquery.storage.v1.BigQueryRead.*'
@@ -37,10 +24,21 @@ authentication:
3724
publishing:
3825
new_issue_uri: https://issuetracker.google.com/issues/new?component=187149&template=1162659
3926
documentation_uri: https://cloud.google.com/bigquery/docs/reference/storage/
27+
api_short_name: bigquerystorage
28+
github_label: 'api: bigquerystorage'
29+
doc_tag_prefix: bigquerystorage
30+
organization: CLOUD
4031
library_settings:
4132
- version: google.cloud.bigquery.storage.v1
4233
launch_stage: GA
4334
java_settings:
4435
library_package: com.google.cloud.bigquery.storage.v1
4536
service_class_names:
4637
google.cloud.bigquery.storage.v1.BigQueryRead: BaseBigQueryRead
38+
common:
39+
destinations:
40+
- PACKAGE_MANAGER
41+
python_settings:
42+
common:
43+
destinations:
44+
- PACKAGE_MANAGER

‎google/cloud/bigquery/storage/v1/storage.proto‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,9 @@ service BigQueryRead {
7878
}
7979

8080
// Reads rows from the stream in the format prescribed by the ReadSession.
81-
// Each response contains one or more table rows, up to a maximum of 100 MiB
81+
// Each response contains one or more table rows, up to a maximum of 128 MB
8282
// per response; read requests which attempt to read individual rows larger
83-
// than 100 MiB will fail.
83+
// than 128 MB will fail.
8484
//
8585
// Each request also returns a set of stream statistics reflecting the current
8686
// state of the stream.
@@ -423,8 +423,6 @@ message CreateWriteStreamRequest {
423423
// Requests larger than this return an error, typically `INVALID_ARGUMENT`.
424424
message AppendRowsRequest {
425425
// Arrow schema and data.
426-
// Arrow format is an experimental feature only selected for allowlisted
427-
// customers.
428426
message ArrowData {
429427
// Optional. Arrow Schema used to serialize the data.
430428
ArrowSchema writer_schema = 1;
@@ -436,8 +434,8 @@ message AppendRowsRequest {
436434
// ProtoData contains the data rows and schema when constructing append
437435
// requests.
438436
message ProtoData {
439-
// The protocol buffer schema used to serialize the data. Provide this value
440-
// whenever:
437+
// Optional. The protocol buffer schema used to serialize the data. Provide
438+
// this value whenever:
441439
//
442440
// * You send the first request of an RPC connection.
443441
//
@@ -446,7 +444,7 @@ message AppendRowsRequest {
446444
// * You specify a new destination table.
447445
ProtoSchema writer_schema = 1;
448446

449-
// Serialized row data in protobuf message format.
447+
// Required. Serialized row data in protobuf message format.
450448
// Currently, the backend expects the serialized rows to adhere to
451449
// proto2 semantics when appending rows, particularly with respect to
452450
// how default values are encoded.
@@ -522,8 +520,7 @@ message AppendRowsRequest {
522520
// Rows in proto format.
523521
ProtoData proto_rows = 4;
524522

525-
// Rows in arrow format. This is an experimental feature only selected for
526-
// allowlisted customers.
523+
// Rows in arrow format.
527524
ArrowData arrow_rows = 5;
528525
}
529526

@@ -553,8 +550,8 @@ message AppendRowsRequest {
553550

554551
// Optional. Default missing value interpretation for all columns in the
555552
// table. When a value is specified on an `AppendRowsRequest`, it is applied
556-
// to all requests on the connection from that point forward, until a
557-
// subsequent `AppendRowsRequest` sets it to a different value.
553+
// to all requests from that point forward, until a subsequent
554+
// `AppendRowsRequest` sets it to a different value.
558555
// `missing_value_interpretation` can override
559556
// `default_missing_value_interpretation`. For example, if you want to write
560557
// `NULL` instead of using default values for some columns, you can set

‎google/cloud/bigquery/storage/v1/stream.proto‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,8 +328,8 @@ message WriteStream {
328328
// Immutable. Mode of the stream.
329329
WriteMode write_mode = 7 [(google.api.field_behavior) = IMMUTABLE];
330330

331-
// Immutable. The geographic location where the stream's dataset resides. See
332-
// https://cloud.google.com/bigquery/docs/locations for supported
331+
// Output only. The geographic location where the stream's dataset resides.
332+
// See https://cloud.google.com/bigquery/docs/locations for supported
333333
// locations.
334-
string location = 8 [(google.api.field_behavior) = IMMUTABLE];
334+
string location = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
335335
}

‎google/cloud/bigquery/storage/v1/table.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ syntax = "proto3";
1717
package google.cloud.bigquery.storage.v1;
1818

1919
import "google/api/field_behavior.proto";
20+
import "google/protobuf/wrappers.proto";
2021

2122
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
2223
option go_package = "cloud.google.com/go/bigquery/storage/apiv1/storagepb;storagepb";
@@ -178,6 +179,16 @@ message TableFieldSchema {
178179
// (https://cloud.google.com/bigquery/docs/default-values) for this field.
179180
string default_value_expression = 10 [(google.api.field_behavior) = OPTIONAL];
180181

182+
// Optional. Precision (maximum number of total digits in base 10) for seconds
183+
// of TIMESTAMP type.
184+
//
185+
// Possible values include:
186+
//
187+
// * 6 (Default, for TIMESTAMP type with microsecond precision)
188+
// * 12 (For TIMESTAMP type with picosecond precision)
189+
google.protobuf.Int64Value timestamp_precision = 27
190+
[(google.api.field_behavior) = OPTIONAL];
191+
181192
// Optional. The subtype of the RANGE, if the type of this field is RANGE. If
182193
// the type is RANGE, this field is required. Possible values for the field
183194
// element type of a RANGE include:

0 commit comments

Comments
 (0)