Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit baf8fb3

Browse files
yirutangyirutang
authored
feat: writeapi v1 manual client lib (#1323)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/java-bigquerystorage/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #<issue_number_goes_here> ☕️
1 parent e33f388 commit baf8fb3

20 files changed

Lines changed: 6086 additions & 0 deletions

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1;
17+
18+
import com.google.common.base.Preconditions;
19+
import com.google.common.collect.ImmutableList;
20+
import com.google.common.collect.ImmutableMap;
21+
import com.google.protobuf.DescriptorProtos.DescriptorProto;
22+
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
23+
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
24+
import com.google.protobuf.Descriptors;
25+
import com.google.protobuf.Descriptors.Descriptor;
26+
import com.google.protobuf.Descriptors.FileDescriptor;
27+
import java.util.ArrayList;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
31+
/**
32+
* Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase
33+
* when constructing the protobuf descriptor. The mapping between field types and field modes are
34+
* shown in the ImmutableMaps below.
35+
*/
36+
public class BQTableSchemaToProtoDescriptor {
37+
private static ImmutableMap<TableFieldSchema.Mode, FieldDescriptorProto.Label>
38+
BQTableSchemaModeMap =
39+
ImmutableMap.of(
40+
TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL,
41+
TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED,
42+
TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED);
43+
44+
private static ImmutableMap<TableFieldSchema.Type, FieldDescriptorProto.Type>
45+
BQTableSchemaTypeMap =
46+
new ImmutableMap.Builder<TableFieldSchema.Type, FieldDescriptorProto.Type>()
47+
.put(TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL)
48+
.put(TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES)
49+
.put(TableFieldSchema.Type.DATE, FieldDescriptorProto.Type.TYPE_INT32)
50+
.put(TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_INT64)
51+
.put(TableFieldSchema.Type.DOUBLE, FieldDescriptorProto.Type.TYPE_DOUBLE)
52+
.put(TableFieldSchema.Type.GEOGRAPHY, FieldDescriptorProto.Type.TYPE_STRING)
53+
.put(TableFieldSchema.Type.INT64, FieldDescriptorProto.Type.TYPE_INT64)
54+
.put(TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_BYTES)
55+
.put(TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
56+
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
57+
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
58+
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
59+
.build();
60+
61+
/**
62+
* Converts TableFieldSchema to a Descriptors.Descriptor object.
63+
*
64+
* @param BQTableSchema
65+
* @throws Descriptors.DescriptorValidationException
66+
*/
67+
public static Descriptor convertBQTableSchemaToProtoDescriptor(TableSchema BQTableSchema)
68+
throws Descriptors.DescriptorValidationException {
69+
Preconditions.checkNotNull(BQTableSchema, "BQTableSchema is null.");
70+
return convertBQTableSchemaToProtoDescriptorImpl(
71+
BQTableSchema, "root", new HashMap<ImmutableList<TableFieldSchema>, Descriptor>());
72+
}
73+
74+
/**
75+
* Converts a TableFieldSchema to a Descriptors.Descriptor object.
76+
*
77+
* @param BQTableSchema
78+
* @param scope Keeps track of current scope to prevent repeated naming while constructing
79+
* descriptor.
80+
* @param dependencyMap Stores already constructed descriptors to prevent reconstruction
81+
* @throws Descriptors.DescriptorValidationException
82+
*/
83+
private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
84+
TableSchema BQTableSchema,
85+
String scope,
86+
HashMap<ImmutableList<TableFieldSchema>, Descriptor> dependencyMap)
87+
throws Descriptors.DescriptorValidationException {
88+
List<FileDescriptor> dependenciesList = new ArrayList<FileDescriptor>();
89+
List<FieldDescriptorProto> fields = new ArrayList<FieldDescriptorProto>();
90+
int index = 1;
91+
for (TableFieldSchema BQTableField : BQTableSchema.getFieldsList()) {
92+
String currentScope = scope + "__" + BQTableField.getName();
93+
if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
94+
ImmutableList<TableFieldSchema> fieldList =
95+
ImmutableList.copyOf(BQTableField.getFieldsList());
96+
if (dependencyMap.containsKey(fieldList)) {
97+
Descriptor descriptor = dependencyMap.get(fieldList);
98+
dependenciesList.add(descriptor.getFile());
99+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, descriptor.getName()));
100+
} else {
101+
Descriptor descriptor =
102+
convertBQTableSchemaToProtoDescriptorImpl(
103+
TableSchema.newBuilder().addAllFields(fieldList).build(),
104+
currentScope,
105+
dependencyMap);
106+
dependenciesList.add(descriptor.getFile());
107+
dependencyMap.put(fieldList, descriptor);
108+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
109+
}
110+
} else {
111+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
112+
}
113+
}
114+
FileDescriptor[] dependenciesArray = new FileDescriptor[dependenciesList.size()];
115+
dependenciesArray = dependenciesList.toArray(dependenciesArray);
116+
DescriptorProto descriptorProto =
117+
DescriptorProto.newBuilder().setName(scope).addAllField(fields).build();
118+
FileDescriptorProto fileDescriptorProto =
119+
FileDescriptorProto.newBuilder().addMessageType(descriptorProto).build();
120+
FileDescriptor fileDescriptor =
121+
FileDescriptor.buildFrom(fileDescriptorProto, dependenciesArray);
122+
Descriptor descriptor = fileDescriptor.findMessageTypeByName(scope);
123+
return descriptor;
124+
}
125+
126+
/**
127+
* Converts a BQTableField to ProtoField
128+
*
129+
* @param BQTableField BQ Field used to construct a FieldDescriptorProto
130+
* @param index Index for protobuf fields.
131+
* @param scope used to name descriptors
132+
*/
133+
private static FieldDescriptorProto convertBQTableFieldToProtoField(
134+
TableFieldSchema BQTableField, int index, String scope) {
135+
TableFieldSchema.Mode mode = BQTableField.getMode();
136+
String fieldName = BQTableField.getName().toLowerCase();
137+
if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
138+
return FieldDescriptorProto.newBuilder()
139+
.setName(fieldName)
140+
.setTypeName(scope)
141+
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode))
142+
.setNumber(index)
143+
.build();
144+
}
145+
return FieldDescriptorProto.newBuilder()
146+
.setName(fieldName)
147+
.setType((FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()))
148+
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode))
149+
.setNumber(index)
150+
.build();
151+
}
152+
}

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BigDecimalByteStringEncoder.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* This code was ported from ZetaSQL and can be found here:
19+
* https://github.com/google/zetasql/blob/c55f967a5ae35b476437210c529691d8a73f5507/java/com/google/zetasql/Value.java
20+
*/
21+
22+
package com.google.cloud.bigquery.storage.v1;
23+
24+
import com.google.common.primitives.Bytes;
25+
import com.google.protobuf.ByteString;
26+
import java.math.BigDecimal;
27+
import java.math.BigInteger;
28+
29+
public class BigDecimalByteStringEncoder {
30+
private static int NumericScale = 9;
31+
private static final BigDecimal MAX_NUMERIC_VALUE =
32+
new BigDecimal("99999999999999999999999999999.999999999");
33+
private static final BigDecimal MIN_NUMERIC_VALUE =
34+
new BigDecimal("-99999999999999999999999999999.999999999");
35+
36+
public static ByteString encodeToNumericByteString(BigDecimal bigDecimal) {
37+
ByteString byteString =
38+
serializeBigDecimal(
39+
bigDecimal, NumericScale, MAX_NUMERIC_VALUE, MIN_NUMERIC_VALUE, "ByteString");
40+
return byteString;
41+
}
42+
43+
public static BigDecimal decodeNumericByteString(ByteString byteString) {
44+
BigDecimal bigDecimal =
45+
deserializeBigDecimal(
46+
byteString, NumericScale, MAX_NUMERIC_VALUE, MIN_NUMERIC_VALUE, "BigDecimal");
47+
return bigDecimal;
48+
}
49+
// Make these private and make public wrapper that internalizes these min/max/scale/type
50+
private static BigDecimal deserializeBigDecimal(
51+
ByteString serializedValue,
52+
int scale,
53+
BigDecimal maxValue,
54+
BigDecimal minValue,
55+
String typeName) {
56+
byte[] bytes = serializedValue.toByteArray();
57+
// NUMERIC/BIGNUMERIC values are serialized as scaled integers in two's complement form in
58+
// little endian order. BigInteger requires the same encoding but in big endian order,
59+
// therefore we must reverse the bytes that come from the proto.
60+
Bytes.reverse(bytes);
61+
BigInteger scaledValue = new BigInteger(bytes);
62+
BigDecimal decimalValue = new BigDecimal(scaledValue, scale);
63+
if (decimalValue.compareTo(maxValue) > 0 || decimalValue.compareTo(minValue) < 0) {
64+
throw new IllegalArgumentException(typeName + " overflow: " + decimalValue.toPlainString());
65+
}
66+
return decimalValue;
67+
}
68+
/** Returns a numeric Value that equals to {@code v}. */
69+
private static ByteString serializeBigDecimal(
70+
BigDecimal v, int scale, BigDecimal maxValue, BigDecimal minValue, String typeName) {
71+
if (v.scale() > scale) {
72+
throw new IllegalArgumentException(
73+
typeName + " scale cannot exceed " + scale + ": " + v.toPlainString());
74+
}
75+
if (v.compareTo(maxValue) > 0 || v.compareTo(minValue) < 0) {
76+
throw new IllegalArgumentException(typeName + " overflow: " + v.toPlainString());
77+
}
78+
byte[] bytes = v.setScale(scale).unscaledValue().toByteArray();
79+
// NUMERIC/BIGNUMERIC values are serialized as scaled integers in two's complement form in
80+
// little endian
81+
// order. BigInteger requires the same encoding but in big endian order, therefore we must
82+
// reverse the bytes that come from the proto.
83+
Bytes.reverse(bytes);
84+
return ByteString.copyFrom(bytes);
85+
}
86+
}

0 commit comments

Comments
 (0)