fix: Ignore utf8 keys. (#19643)

marianogappa · web-flow · commit 411474ab176c · 2024-11-20T07:59:34.000Z
fixes cloudquery/cloudquery-issues#2763 (internal issue) Normally, the JSON Flattener will attempt to flatten JSON objects to the first level, e.g.: ```json "type_schema": "[{\"availability_domain\":\"utf8\",\"display_name\":\"utf8\",\"volume_group_replica_id\":\"utf8\"}]" ``` But the TypeSchema notation had a special case that the JSON Flattener wasn't aware of: ```json { "name": "freeform_tags", "type": "json", "type_schema": "{\"utf8\":\"utf8\"}" }, ``` In this case, the `utf8` key is not a literal string, but a special value that means "any string key". This should be ignored by the flattener, because it's meant to contain different keys on different rows, and table schemas are fixed. The only way to implement this would be to do a linear pass over all rows and learn all possible keys, then do another pass and create nullable columns on all non-supplied columns for each row. We definitely don't want to support this. Instead, this PR simply ignores keys with this special literal string. The practice is common on the Oracle source plugin; here's a flattened Oracle table that was suffering from this issue, now fixed: <img width="561" alt="Screenshot 2024-11-20 at 10 14 43" src="https://github.com/user-attachments/assets/33013665-eb4f-4191-8e50-53004471d43a">
diff --git a/plugins/transformer/jsonflattener/client/recordupdater/record_updater.go b/plugins/transformer/jsonflattener/client/recordupdater/record_updater.go
@@ -160,6 +160,12 @@ func (*RecordUpdater) preprocessRow(colName string, rowIndex int, rawRow any) (m
 func preprocessTypeSchema(unprocessedTypeSchema map[string]any) map[string]string {
 	typeSchema := make(map[string]string)
 	for key, typ := range unprocessedTypeSchema {
+		// Edge case: if the key is utf8, we don't process it, because utf8 is a special
+		// string that means that there can be many keys with any name.
+		if key == "utf8" {
+			continue
+		}
+
 		// If the type of a given key is not string, we consider it as a JSON type
 		// so that we don't flatten deeper than the first level.
 		if _, ok := typ.(string); !ok {
diff --git a/plugins/transformer/jsonflattener/client/recordupdater/record_updater_test.go b/plugins/transformer/jsonflattener/client/recordupdater/record_updater_test.go
@@ -33,6 +33,26 @@ func TestFlattenJSONFields(t *testing.T) {
 	require.Equal(t, true, updatedRecord.Column(3).(*array.Boolean).Value(0))
 }
 
+func TestFlattenJSONFieldsDoesntFlattenFieldsKeyedUTF8(t *testing.T) {
+	record := testRecord(
+		[]string{"col1"},
+		map[string]string{"col1": `{"key_a": "utf8", "key_b": "int64", "utf8": "any"}`},
+		[]arrow.Array{buildJSONColumn([]*any{toP(`{"key_a": "value", "key_b": 2, "utf8": "any"}`)})},
+	)
+	updater := New(record)
+
+	updatedRecord, err := updater.FlattenJSONFields()
+	require.NoError(t, err)
+
+	require.Equal(t, int64(3), updatedRecord.NumCols())
+	require.Equal(t, int64(1), updatedRecord.NumRows())
+	requireAllColsLenMatchRecordsLen(t, updatedRecord)
+	require.Equal(t, "col1__key_a", updatedRecord.ColumnName(1))
+	require.Equal(t, "value", updatedRecord.Column(1).(*array.String).Value(0))
+	require.Equal(t, "col1__key_b", updatedRecord.ColumnName(2))
+	require.Equal(t, int64(2), updatedRecord.Column(2).(*array.Int64).Value(0))
+}
+
 func TestNestedJSONFlattenedToFirstLevel(t *testing.T) {
 	record := testRecord(
 		[]string{"col1"},
