Skip to content

Commit 0a1764f

Browse files
candiduslynxcandiduslynx
authored
fix(deps): Update github.com/cloudquery/plugin-sdk/v3 to v3.8.1 (#11078)
Extracted from #11043 Depends on: * cloudquery/plugin-sdk#913 * apache/arrow#35823 * cloudquery/plugin-sdk#921
1 parent ed61053 commit 0a1764f

13 files changed

Lines changed: 195 additions & 93 deletions

File tree

‎plugins/destination/clickhouse/client/client_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"os"
66
"testing"
77

8+
"github.com/apache/arrow/go/v13/arrow"
89
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/resources/plugin"
910
"github.com/cloudquery/plugin-pb-go/specs"
1011
"github.com/cloudquery/plugin-sdk/v3/plugins/destination"
@@ -52,5 +53,17 @@ func TestPlugin(t *testing.T) {
5253

5354
MigrateStrategyAppend: migrateStrategy,
5455
},
56+
destination.WithTestSourceAllowNull(func(dt arrow.DataType) bool {
57+
switch dt.(type) {
58+
case *arrow.StructType,
59+
*arrow.MapType,
60+
*arrow.ListType,
61+
*arrow.LargeListType,
62+
*arrow.FixedSizeListType:
63+
return false
64+
default:
65+
return true
66+
}
67+
}),
5568
)
5669
}

‎plugins/destination/clickhouse/go.mod‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,21 @@ module github.com/cloudquery/cloudquery/plugins/destination/clickhouse
33
go 1.19
44

55
require (
6-
github.com/ClickHouse/clickhouse-go/v2 v2.9.2
6+
github.com/ClickHouse/clickhouse-go/v2 v2.10.0
77
github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
88
github.com/cloudquery/plugin-pb-go v1.0.8
9-
github.com/cloudquery/plugin-sdk/v3 v3.5.2
9+
github.com/cloudquery/plugin-sdk/v3 v3.8.1
1010
github.com/google/uuid v1.3.0
1111
github.com/rs/zerolog v1.29.1
12-
github.com/stretchr/testify v1.8.2
12+
github.com/stretchr/testify v1.8.3
1313
golang.org/x/sync v0.2.0
1414
)
1515

16-
require google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
17-
18-
// TODO: remove once al updates are merged
19-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
16+
// TODO: remove once all updates are merged
17+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed
2018

2119
require (
22-
github.com/ClickHouse/ch-go v0.55.0 // indirect
20+
github.com/ClickHouse/ch-go v0.56.0 // indirect
2321
github.com/andybalholm/brotli v1.0.5 // indirect
2422
github.com/apache/thrift v0.18.1 // indirect
2523
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
@@ -53,15 +51,16 @@ require (
5351
github.com/spf13/pflag v1.0.5 // indirect
5452
github.com/thoas/go-funk v0.9.3 // indirect
5553
github.com/zeebo/xxh3 v1.0.2 // indirect
56-
go.opentelemetry.io/otel v1.15.1 // indirect
57-
go.opentelemetry.io/otel/trace v1.15.1 // indirect
54+
go.opentelemetry.io/otel v1.16.0 // indirect
55+
go.opentelemetry.io/otel/trace v1.16.0 // indirect
5856
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
5957
golang.org/x/mod v0.10.0 // indirect
6058
golang.org/x/net v0.10.0 // indirect
6159
golang.org/x/sys v0.8.0 // indirect
6260
golang.org/x/text v0.9.0 // indirect
6361
golang.org/x/tools v0.9.0 // indirect
6462
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
63+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
6564
google.golang.org/grpc v1.55.0 // indirect
6665
google.golang.org/protobuf v1.30.0 // indirect
6766
gopkg.in/yaml.v2 v2.4.0 // indirect

‎plugins/destination/clickhouse/go.sum‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
3333
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
3434
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
3535
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
36-
github.com/ClickHouse/ch-go v0.55.0 h1:jw4Tpx887YXrkyL5DfgUome/po8MLz92nz2heOQ6RjQ=
37-
github.com/ClickHouse/ch-go v0.55.0/go.mod h1:kQT2f+yp2p+sagQA/7kS6G3ukym+GQ5KAu1kuFAFDiU=
38-
github.com/ClickHouse/clickhouse-go/v2 v2.9.2 h1:P9az39xLJGwdL+Bq04Qrcq0lJspTgGT0VD7ESouFOYg=
39-
github.com/ClickHouse/clickhouse-go/v2 v2.9.2/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg=
36+
github.com/ClickHouse/ch-go v0.56.0 h1:Uli/jOQl7+pYyA2E+/VvOSQRX3jHZ4FjESCRReAeFuM=
37+
github.com/ClickHouse/ch-go v0.56.0/go.mod h1:nKtI66Lz8pRvbO7NZDrgbRaC9ldsD4HMeWKb1hpbpBI=
38+
github.com/ClickHouse/clickhouse-go/v2 v2.10.0 h1:0w/A50D5MfsRUYBaV6rLKwZ4LXWKLZKJ1u31QXjTIO4=
39+
github.com/ClickHouse/clickhouse-go/v2 v2.10.0/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg=
4040
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
4141
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
4242
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
@@ -48,14 +48,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
4848
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
4949
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
5050
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
51-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
52-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
51+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed h1:FQPwpdGI7TIRUfgn7WgVSx9PCa8/kmelveY6FFqriAM=
52+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230601133906-ff55c1eab1ed/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
5353
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
5454
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5555
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5656
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
57-
github.com/cloudquery/plugin-sdk/v3 v3.5.2 h1:Ss8m1/T3tY4IFFo4ZRku4fCJEOOXILe33SUGB5sfpvk=
58-
github.com/cloudquery/plugin-sdk/v3 v3.5.2/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
57+
github.com/cloudquery/plugin-sdk/v3 v3.8.1 h1:Rj+3zBLmscKSbG+JPLT5bzgv56oPwBHRSMGyJ1DSfyc=
58+
github.com/cloudquery/plugin-sdk/v3 v3.8.1/go.mod h1:8PHS8cMjWPeXrurnI30dyHwViK4HJUZLA6uys+F2fXQ=
5959
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
6060
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
6161
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -218,16 +218,12 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM
218218
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
219219
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
220220
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
221-
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
222-
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
223221
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
224222
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
225223
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
226224
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
227-
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
228-
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
229-
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
230-
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
225+
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
226+
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
231227
github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
232228
github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
233229
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -248,10 +244,10 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
248244
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
249245
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
250246
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
251-
go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8=
252-
go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc=
253-
go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY=
254-
go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8=
247+
go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
248+
go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
249+
go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
250+
go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
255251
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
256252
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
257253
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -495,8 +491,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
495491
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
496492
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
497493
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
498-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
499-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
494+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
495+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
500496
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
501497
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
502498
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

‎plugins/destination/clickhouse/typeconv/arrow/values/map.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ func buildMapFromReflect(builder *array.MapBuilder, value reflect.Value) error {
4040
return nil
4141
}
4242

43+
builder.Append(true)
4344
keyBuilder, itemBuilder := builder.KeyBuilder(), builder.ItemBuilder()
4445
it := value.MapRange()
4546
for it.Next() {
46-
builder.Append(true)
4747
if err := buildValue(keyBuilder, it.Key().Interface()); err != nil {
4848
return err
4949
}

‎plugins/destination/clickhouse/typeconv/arrow/values/string.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,13 @@ func buildFromString(builder array.Builder, value any) error {
1010
builder.AppendNull()
1111
return nil
1212
}
13-
return builder.AppendValueFromString(v)
13+
14+
if len(v) > 0 {
15+
return builder.AppendValueFromString(v)
16+
}
17+
18+
// binary types are handled separately, so here we have a builder that most likely can't handle empty string.
19+
// having empty string in CH means that this was an empty value
20+
builder.AppendEmptyValue()
21+
return nil
1422
}

‎plugins/destination/clickhouse/typeconv/arrow/values/time.go‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package values
22

33
import (
4-
"fmt"
54
"time"
65

76
"github.com/apache/arrow/go/v13/arrow"
@@ -12,29 +11,52 @@ func buildDate32Values(builder primitiveBuilder[arrow.Date32], value any) {
1211
v, ok := unwrap[time.Time](value)
1312
if !ok {
1413
builder.AppendNull()
14+
return
1515
}
16+
17+
if v.IsZero() {
18+
// work-around for empty values
19+
builder.AppendEmptyValue()
20+
return
21+
}
22+
1623
builder.Append(arrow.Date32FromTime(v))
1724
}
1825

1926
func buildDate64Values(builder primitiveBuilder[arrow.Date64], value any) {
2027
v, ok := unwrap[time.Time](value)
2128
if !ok {
2229
builder.AppendNull()
30+
return
31+
}
32+
33+
if v.IsZero() {
34+
// work-around for empty values
35+
builder.AppendEmptyValue()
36+
return
2337
}
38+
2439
builder.Append(arrow.Date64FromTime(v))
2540
}
2641

2742
func buildTimestampValues(builder *array.TimestampBuilder, value any) error {
2843
v, ok := unwrap[time.Time](value)
2944
if !ok {
3045
builder.AppendNull()
46+
return nil
3147
}
3248

3349
t, err := timeToTimestamp(v, builder.Type().(*arrow.TimestampType))
3450
if err != nil {
3551
return err
3652
}
3753

54+
if v.IsZero() {
55+
// work-around for empty values
56+
builder.AppendEmptyValue()
57+
return nil
58+
}
59+
3860
builder.Append(t)
3961
return nil
4062
}
@@ -48,16 +70,5 @@ func timeToTimestamp(value time.Time, tsType *arrow.TimestampType) (arrow.Timest
4870
value = value.In(loc)
4971
}
5072

51-
switch tsType.Unit {
52-
case arrow.Second:
53-
return arrow.Timestamp(value.Unix()), nil
54-
case arrow.Millisecond:
55-
return arrow.Timestamp(value.Unix()*1e3 + int64(value.Nanosecond())/1e6), nil
56-
case arrow.Microsecond:
57-
return arrow.Timestamp(value.Unix()*1e6 + int64(value.Nanosecond())/1e3), nil
58-
case arrow.Nanosecond:
59-
return arrow.Timestamp(value.UnixNano()), nil
60-
default:
61-
return arrow.Timestamp(0), fmt.Errorf("unsupported Apache Arrow time unit: %s", tsType.Unit.String())
62-
}
73+
return arrow.TimestampFromTime(value, tsType.Unit)
6374
}

‎plugins/destination/clickhouse/typeconv/ch/values/list.go‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"time"
66

77
"github.com/ClickHouse/clickhouse-go/v2/lib/column"
8+
"github.com/apache/arrow/go/v13/arrow"
89
"github.com/apache/arrow/go/v13/arrow/array"
10+
"github.com/apache/arrow/go/v13/arrow/memory"
911
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/typeconv/ch/types"
1012
)
1113

@@ -22,12 +24,14 @@ func listValue(arr array.ListLike) (any, error) {
2224
}
2325
valueType := col.ScanType()
2426

27+
sanitized, err := sanitizeNested(arr)
28+
if err != nil {
29+
return nil, err
30+
}
31+
arr = sanitized.(array.ListLike)
32+
2533
elems := make([]any, arr.Len())
2634
for i := 0; i < arr.Len(); i++ {
27-
if arr.IsNull(i) {
28-
continue
29-
}
30-
3135
from, to := arr.ValueOffsets(i)
3236
elems[i], err = FromArray(array.NewSlice(arr.ListValues(), from, to))
3337
if err != nil {
@@ -37,13 +41,33 @@ func listValue(arr array.ListLike) (any, error) {
3741

3842
res := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(valueType)), len(elems), len(elems)) // we do []*(type) for nullable assignment
3943
for i, elem := range elems {
40-
// we need to fill in for the in-depth recursive parsing by ClickHouse SDK
44+
// lists aren't nullable themselves
45+
// https://clickhouse.com/docs/en/sql-reference/data-types/nullable
4146
val := reflect.New(valueType)
42-
if elem != nil {
43-
val.Elem().Set(reflect.ValueOf(elem))
44-
}
47+
val.Elem().Set(reflect.ValueOf(elem))
4548
res.Index(i).Set(val)
4649
}
4750

4851
return res.Interface(), nil
4952
}
53+
54+
// sanitizeNested will replace all null entries with empty ones as in CH nested types aren't nullable themselves
55+
// https://clickhouse.com/docs/en/sql-reference/data-types/nullable
56+
func sanitizeNested(arr arrow.Array) (arrow.Array, error) {
57+
if arr.NullN() == 0 {
58+
return arr, nil
59+
}
60+
61+
builder := array.NewBuilder(memory.DefaultAllocator, arr.DataType())
62+
for i := 0; i < arr.Len(); i++ {
63+
if arr.IsNull(i) {
64+
builder.AppendEmptyValue()
65+
continue
66+
}
67+
if err := builder.AppendValueFromString(arr.ValueStr(i)); err != nil {
68+
return nil, err
69+
}
70+
}
71+
72+
return builder.NewArray(), nil
73+
}

0 commit comments

Comments
 (0)