Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit ab6213c

Browse files
yirutangyirutang
authored
fix: Add special type tests for Json writer and fix some type mapping issues (#725)
* fix: a race condition in test * . * fix: offset_sets to offsetSets in jsonwritertest * fix: add test coverage for special type handling in JSON object * . * . * . * . * . * . * . * .
1 parent f4de72c commit ab6213c

7 files changed

Lines changed: 138 additions & 30 deletions

File tree

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ public class BQTableSchemaToProtoDescriptor {
4747
.put(Table.TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL)
4848
.put(Table.TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES)
4949
.put(Table.TableFieldSchema.Type.DATE, FieldDescriptorProto.Type.TYPE_INT32)
50-
.put(Table.TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_INT64)
50+
.put(Table.TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_STRING)
5151
.put(Table.TableFieldSchema.Type.DOUBLE, FieldDescriptorProto.Type.TYPE_DOUBLE)
5252
.put(Table.TableFieldSchema.Type.GEOGRAPHY, FieldDescriptorProto.Type.TYPE_STRING)
5353
.put(Table.TableFieldSchema.Type.INT64, FieldDescriptorProto.Type.TYPE_INT64)
54-
.put(Table.TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_BYTES)
54+
.put(Table.TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_STRING)
5555
.put(Table.TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
5656
.put(Table.TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
57-
.put(Table.TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
57+
.put(Table.TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_STRING)
5858
.put(Table.TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
5959
.build();
6060

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ public class BQTableSchemaToProtoDescriptor {
4747
.put(TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL)
4848
.put(TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES)
4949
.put(TableFieldSchema.Type.DATE, FieldDescriptorProto.Type.TYPE_INT32)
50-
.put(TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_INT64)
50+
.put(TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_STRING)
5151
.put(TableFieldSchema.Type.DOUBLE, FieldDescriptorProto.Type.TYPE_DOUBLE)
5252
.put(TableFieldSchema.Type.GEOGRAPHY, FieldDescriptorProto.Type.TYPE_STRING)
5353
.put(TableFieldSchema.Type.INT64, FieldDescriptorProto.Type.TYPE_INT64)
54-
.put(TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_BYTES)
54+
.put(TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_STRING)
5555
.put(TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
5656
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
57-
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
57+
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_STRING)
5858
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
5959
.build();
6060

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,13 @@ public class BQTableSchemaToProtoDescriptorTest {
3939
.put(Table.TableFieldSchema.Type.BOOL, BoolType.getDescriptor())
4040
.put(Table.TableFieldSchema.Type.BYTES, BytesType.getDescriptor())
4141
.put(Table.TableFieldSchema.Type.DATE, Int32Type.getDescriptor())
42-
.put(Table.TableFieldSchema.Type.DATETIME, Int64Type.getDescriptor())
42+
.put(Table.TableFieldSchema.Type.DATETIME, StringType.getDescriptor())
4343
.put(Table.TableFieldSchema.Type.DOUBLE, DoubleType.getDescriptor())
4444
.put(Table.TableFieldSchema.Type.GEOGRAPHY, StringType.getDescriptor())
4545
.put(Table.TableFieldSchema.Type.INT64, Int64Type.getDescriptor())
46-
.put(Table.TableFieldSchema.Type.NUMERIC, BytesType.getDescriptor())
46+
.put(Table.TableFieldSchema.Type.NUMERIC, StringType.getDescriptor())
4747
.put(Table.TableFieldSchema.Type.STRING, StringType.getDescriptor())
48-
.put(Table.TableFieldSchema.Type.TIME, Int64Type.getDescriptor())
48+
.put(Table.TableFieldSchema.Type.TIME, StringType.getDescriptor())
4949
.put(Table.TableFieldSchema.Type.TIMESTAMP, Int64Type.getDescriptor())
5050
.build();
5151

@@ -75,7 +75,7 @@ private void isDescriptorEqual(Descriptor convertedProto, Descriptor originalPro
7575
// Check type
7676
FieldDescriptor.Type convertedType = convertedField.getType();
7777
FieldDescriptor.Type originalType = originalField.getType();
78-
assertEquals(convertedType, originalType);
78+
assertEquals(convertedField.getName(), convertedType, originalType);
7979
// Check mode
8080
assertTrue(
8181
(originalField.isRepeated() == convertedField.isRepeated())
@@ -181,6 +181,30 @@ public void testStructComplex() throws Exception {
181181
.addFields(1, ComplexLvl2)
182182
.setName("complex_lvl1")
183183
.build();
184+
final Table.TableFieldSchema TEST_NUMERIC =
185+
Table.TableFieldSchema.newBuilder()
186+
.setType(Table.TableFieldSchema.Type.NUMERIC)
187+
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
188+
.setName("test_numeric")
189+
.build();
190+
final Table.TableFieldSchema TEST_GEO =
191+
Table.TableFieldSchema.newBuilder()
192+
.setType(Table.TableFieldSchema.Type.GEOGRAPHY)
193+
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
194+
.setName("test_geo")
195+
.build();
196+
final Table.TableFieldSchema TEST_TIMESTAMP =
197+
Table.TableFieldSchema.newBuilder()
198+
.setType(Table.TableFieldSchema.Type.TIMESTAMP)
199+
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
200+
.setName("test_timestamp")
201+
.build();
202+
final Table.TableFieldSchema TEST_TIME =
203+
Table.TableFieldSchema.newBuilder()
204+
.setType(Table.TableFieldSchema.Type.TIME)
205+
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
206+
.setName("test_time")
207+
.build();
184208
final Table.TableSchema tableSchema =
185209
Table.TableSchema.newBuilder()
186210
.addFields(0, test_int)
@@ -191,6 +215,10 @@ public void testStructComplex() throws Exception {
191215
.addFields(5, test_date)
192216
.addFields(6, ComplexLvl1)
193217
.addFields(7, ComplexLvl2)
218+
.addFields(8, TEST_NUMERIC)
219+
.addFields(9, TEST_GEO)
220+
.addFields(10, TEST_TIMESTAMP)
221+
.addFields(11, TEST_TIME)
194222
.build();
195223
final Descriptor descriptor =
196224
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,13 @@ public class BQTableSchemaToProtoDescriptorTest {
3838
.put(TableFieldSchema.Type.BOOL, BoolType.getDescriptor())
3939
.put(TableFieldSchema.Type.BYTES, BytesType.getDescriptor())
4040
.put(TableFieldSchema.Type.DATE, Int32Type.getDescriptor())
41-
.put(TableFieldSchema.Type.DATETIME, Int64Type.getDescriptor())
41+
.put(TableFieldSchema.Type.DATETIME, StringType.getDescriptor())
4242
.put(TableFieldSchema.Type.DOUBLE, DoubleType.getDescriptor())
4343
.put(TableFieldSchema.Type.GEOGRAPHY, StringType.getDescriptor())
4444
.put(TableFieldSchema.Type.INT64, Int64Type.getDescriptor())
45-
.put(TableFieldSchema.Type.NUMERIC, BytesType.getDescriptor())
45+
.put(TableFieldSchema.Type.NUMERIC, StringType.getDescriptor())
4646
.put(TableFieldSchema.Type.STRING, StringType.getDescriptor())
47-
.put(TableFieldSchema.Type.TIME, Int64Type.getDescriptor())
47+
.put(TableFieldSchema.Type.TIME, StringType.getDescriptor())
4848
.put(TableFieldSchema.Type.TIMESTAMP, Int64Type.getDescriptor())
4949
.build();
5050

@@ -74,7 +74,7 @@ private void isDescriptorEqual(Descriptor convertedProto, Descriptor originalPro
7474
// Check type
7575
FieldDescriptor.Type convertedType = convertedField.getType();
7676
FieldDescriptor.Type originalType = originalField.getType();
77-
assertEquals(convertedType, originalType);
77+
assertEquals(convertedField.getName(), convertedType, originalType);
7878
// Check mode
7979
assertTrue(
8080
(originalField.isRepeated() == convertedField.isRepeated())
@@ -179,6 +179,30 @@ public void testStructComplex() throws Exception {
179179
.addFields(1, ComplexLvl2)
180180
.setName("complex_lvl1")
181181
.build();
182+
final TableFieldSchema TEST_NUMERIC =
183+
TableFieldSchema.newBuilder()
184+
.setType(TableFieldSchema.Type.NUMERIC)
185+
.setMode(TableFieldSchema.Mode.NULLABLE)
186+
.setName("test_numeric")
187+
.build();
188+
final TableFieldSchema TEST_GEO =
189+
TableFieldSchema.newBuilder()
190+
.setType(TableFieldSchema.Type.GEOGRAPHY)
191+
.setMode(TableFieldSchema.Mode.NULLABLE)
192+
.setName("test_geo")
193+
.build();
194+
final TableFieldSchema TEST_TIMESTAMP =
195+
TableFieldSchema.newBuilder()
196+
.setType(TableFieldSchema.Type.TIMESTAMP)
197+
.setMode(TableFieldSchema.Mode.NULLABLE)
198+
.setName("test_timestamp")
199+
.build();
200+
final TableFieldSchema TEST_TIME =
201+
TableFieldSchema.newBuilder()
202+
.setType(TableFieldSchema.Type.TIME)
203+
.setMode(TableFieldSchema.Mode.NULLABLE)
204+
.setName("test_time")
205+
.build();
182206
final TableSchema tableSchema =
183207
TableSchema.newBuilder()
184208
.addFields(0, test_int)
@@ -189,6 +213,10 @@ public void testStructComplex() throws Exception {
189213
.addFields(5, test_date)
190214
.addFields(6, ComplexLvl1)
191215
.addFields(7, ComplexLvl2)
216+
.addFields(8, TEST_NUMERIC)
217+
.addFields(9, TEST_GEO)
218+
.addFields(10, TEST_TIMESTAMP)
219+
.addFields(11, TEST_TIME)
192220
.build();
193221
final Descriptor descriptor =
194222
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,30 @@ public class JsonStreamWriterTest {
133133
.addFields(1, COMPLEXLVL2)
134134
.setName("complex_lvl1")
135135
.build();
136+
private final TableFieldSchema TEST_NUMERIC =
137+
TableFieldSchema.newBuilder()
138+
.setType(TableFieldSchema.Type.NUMERIC)
139+
.setMode(TableFieldSchema.Mode.NULLABLE)
140+
.setName("test_numeric")
141+
.build();
142+
private final TableFieldSchema TEST_GEO =
143+
TableFieldSchema.newBuilder()
144+
.setType(TableFieldSchema.Type.GEOGRAPHY)
145+
.setMode(TableFieldSchema.Mode.NULLABLE)
146+
.setName("test_geo")
147+
.build();
148+
private final TableFieldSchema TEST_TIMESTAMP =
149+
TableFieldSchema.newBuilder()
150+
.setType(TableFieldSchema.Type.TIMESTAMP)
151+
.setMode(TableFieldSchema.Mode.NULLABLE)
152+
.setName("test_timestamp")
153+
.build();
154+
private final TableFieldSchema TEST_TIME =
155+
TableFieldSchema.newBuilder()
156+
.setType(TableFieldSchema.Type.TIME)
157+
.setMode(TableFieldSchema.Mode.NULLABLE)
158+
.setName("test_time")
159+
.build();
136160
private final TableSchema COMPLEX_TABLE_SCHEMA =
137161
TableSchema.newBuilder()
138162
.addFields(0, TEST_INT)
@@ -143,6 +167,10 @@ public class JsonStreamWriterTest {
143167
.addFields(5, TEST_DATE)
144168
.addFields(6, COMPLEXLVL1)
145169
.addFields(7, COMPLEXLVL2)
170+
.addFields(8, TEST_NUMERIC)
171+
.addFields(9, TEST_GEO)
172+
.addFields(10, TEST_TIMESTAMP)
173+
.addFields(11, TEST_TIME)
146174
.build();
147175

148176
@Before
@@ -353,6 +381,10 @@ public void testSingleAppendComplexJson() throws Exception {
353381
com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl2.newBuilder()
354382
.setTestInt(3)
355383
.build())
384+
.setTestNumeric("1.23456")
385+
.setTestGeo("POINT(1,1)")
386+
.setTestTimestamp(12345678)
387+
.setTestTime("01:00:01")
356388
.build();
357389
JSONObject complex_lvl2 = new JSONObject();
358390
complex_lvl2.put("test_int", 3);
@@ -370,6 +402,10 @@ public void testSingleAppendComplexJson() throws Exception {
370402
json.put("test_date", 1);
371403
json.put("complex_lvl1", complex_lvl1);
372404
json.put("complex_lvl2", complex_lvl2);
405+
json.put("test_numeric", "1.23456");
406+
json.put("test_geo", "POINT(1,1)");
407+
json.put("test_timestamp", 12345678);
408+
json.put("test_time", "01:00:01");
373409
JSONArray jsonArr = new JSONArray();
374410
jsonArr.put(json);
375411

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/it/ITBigQueryWriteManualClientTest.java‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,15 @@ public void testJsonStreamWriterBatchWriteWithCommittedStream()
216216
TableId.of(DATASET, tableName),
217217
StandardTableDefinition.of(
218218
Schema.of(
219-
com.google.cloud.bigquery.Field.newBuilder("foo", LegacySQLTypeName.STRING)
219+
com.google.cloud.bigquery.Field.newBuilder(
220+
"test_str", StandardSQLTypeName.STRING)
221+
.build(),
222+
com.google.cloud.bigquery.Field.newBuilder(
223+
"test_numerics", StandardSQLTypeName.NUMERIC)
224+
.setMode(Field.Mode.REPEATED)
225+
.build(),
226+
com.google.cloud.bigquery.Field.newBuilder(
227+
"test_datetime", StandardSQLTypeName.DATETIME)
220228
.build())))
221229
.build();
222230
bigquery.create(tableInfo);
@@ -239,28 +247,31 @@ public void testJsonStreamWriterBatchWriteWithCommittedStream()
239247
.build())
240248
.build()) {
241249
LOG.info("Sending one message");
242-
JSONObject foo = new JSONObject();
243-
foo.put("foo", "aaa");
244-
JSONArray jsonArr = new JSONArray();
245-
jsonArr.put(foo);
250+
JSONObject testStr = new JSONObject();
251+
testStr.put("test_str", "aaa");
252+
JSONObject testNumerics = new JSONObject();
253+
testNumerics.put("test_numerics", new JSONArray(new String[] {"123.4", "-9000000"}));
254+
JSONObject testDateTime = new JSONObject();
255+
testDateTime.put("test_datetime", "2020-10-1 12:00:00");
256+
JSONArray row = new JSONArray(new JSONObject[] {testStr, testNumerics, testDateTime});
246257

247258
ApiFuture<AppendRowsResponse> response =
248-
jsonStreamWriter.append(jsonArr, -1, /* allowUnknownFields */ false);
259+
jsonStreamWriter.append(row, -1, /* allowUnknownFields */ false);
249260
assertEquals(0, response.get().getOffset());
250261

251262
LOG.info("Sending two more messages");
252-
JSONObject foo1 = new JSONObject();
253-
foo1.put("foo", "bbb");
254-
JSONObject foo2 = new JSONObject();
255-
foo2.put("foo", "ccc");
263+
JSONObject row1 = new JSONObject();
264+
row1.put("test_str", "bbb");
265+
JSONObject row2 = new JSONObject();
266+
row2.put("test_str", "ccc");
256267
JSONArray jsonArr1 = new JSONArray();
257-
jsonArr1.put(foo1);
258-
jsonArr1.put(foo2);
268+
jsonArr1.put(row1);
269+
jsonArr1.put(row2);
259270

260-
JSONObject foo3 = new JSONObject();
261-
foo3.put("foo", "ddd");
271+
JSONObject row3 = new JSONObject();
272+
row3.put("test_str", "ddd");
262273
JSONArray jsonArr2 = new JSONArray();
263-
jsonArr2.put(foo3);
274+
jsonArr2.put(row3);
264275

265276
ApiFuture<AppendRowsResponse> response1 =
266277
jsonStreamWriter.append(jsonArr1, -1, /* allowUnknownFields */ false);
@@ -276,11 +287,12 @@ public void testJsonStreamWriterBatchWriteWithCommittedStream()
276287
tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L));
277288
Iterator<FieldValueList> iter = result.getValues().iterator();
278289
assertEquals("aaa", iter.next().get(0).getStringValue());
290+
assertEquals("-9000000", iter.next().get(1).getRepeatedValue().get(1).getStringValue());
291+
assertEquals("2020-10-01T12:00:00", iter.next().get(2).getStringValue());
279292
assertEquals("bbb", iter.next().get(0).getStringValue());
280293
assertEquals("ccc", iter.next().get(0).getStringValue());
281294
assertEquals("ddd", iter.next().get(0).getStringValue());
282295
assertEquals(false, iter.hasNext());
283-
jsonStreamWriter.close();
284296
}
285297
}
286298

‎google-cloud-bigquerystorage/src/test/proto/jsonTest.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ message ComplexRoot {
1111
required int32 test_date = 6;
1212
required ComplexLvl1 complex_lvl1 = 7;
1313
required ComplexLvl2 complex_lvl2 = 8;
14+
optional string test_numeric = 9;
15+
optional string test_geo = 10;
16+
optional int64 test_timestamp = 11;
17+
optional string test_time = 12;
1418
}
1519

1620
message CasingComplex {

0 commit comments

Comments
 (0)