Skip to content

Commit 2963399

Browse files
kodiakhq[bot]kodiakhq[bot]
authored
Merge branch 'main' into feat/update_to_plugin_pb
2 parents 78a120d + 2df4413 commit 2963399

12 files changed

Lines changed: 172 additions & 14 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [2.5.4](https://github.com/cloudquery/plugin-sdk/compare/v2.5.3...v2.5.4) (2023-05-05)
9+
10+
11+
### Bug Fixes
12+
13+
* **arrow:** Allow empty and `nil` valid param in `AppendValues` ([#847](https://github.com/cloudquery/plugin-sdk/issues/847)) ([dafd05b](https://github.com/cloudquery/plugin-sdk/commit/dafd05b3e2b8dc406d4b6a4bdaf6d1143e569f1d))
14+
15+
## [2.5.3](https://github.com/cloudquery/plugin-sdk/compare/v2.5.2...v2.5.3) (2023-05-04)
16+
17+
18+
### Bug Fixes
19+
20+
* **arrow:** Add missing table options ([#833](https://github.com/cloudquery/plugin-sdk/issues/833)) ([95a9f0c](https://github.com/cloudquery/plugin-sdk/commit/95a9f0c29c6c2b85fded012341bf00cff0225605))
21+
822
## [2.5.2](https://github.com/cloudquery/plugin-sdk/compare/v2.5.1...v2.5.2) (2023-05-02)
923

1024

‎plugins/destination/plugin_testing_write_append.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ func (s *PluginTestSuite) destinationPluginTestWriteAppend(ctx context.Context,
1818
if err := p.Init(ctx, logger, spec); err != nil {
1919
return fmt.Errorf("failed to init plugin: %w", err)
2020
}
21-
tableName := spec.Name
21+
tableName := fmt.Sprintf("cq_%s_%d", spec.Name, time.Now().Unix())
2222
table := testdata.TestTable(tableName).ToArrowSchema()
2323
syncTime := time.Now().UTC().Round(1 * time.Second)
2424
tables := []*arrow.Schema{

‎schema/arrow.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,10 @@ type MetadataFieldOptions struct {
8383
}
8484

8585
type MetadataSchemaOptions struct {
86-
TableName string
87-
TableDescription string
86+
TableName string
87+
TableDescription string
88+
TableIsIncremental bool
89+
TablePKConstraint string
8890
}
8991

9092
func NewSchemaMetadataFromOptions(opts MetadataSchemaOptions) arrow.Metadata {
@@ -95,6 +97,12 @@ func NewSchemaMetadataFromOptions(opts MetadataSchemaOptions) arrow.Metadata {
9597
if opts.TableDescription != "" {
9698
kv[MetadataTableDescription] = opts.TableDescription
9799
}
100+
if opts.TableIsIncremental {
101+
kv[MetadataIncremental] = MetadataTrue
102+
}
103+
if opts.TablePKConstraint != "" {
104+
kv[MetadataConstraintName] = opts.TablePKConstraint
105+
}
98106
return arrow.MetadataFrom(kv)
99107
}
100108

@@ -295,8 +303,10 @@ func CQSchemaToArrow(table *Table) *arrow.Schema {
295303
fields = append(fields, CQColumnToArrowField(&col))
296304
}
297305
opts := MetadataSchemaOptions{
298-
TableName: table.Name,
299-
TableDescription: table.Description,
306+
TableName: table.Name,
307+
TableDescription: table.Description,
308+
TableIsIncremental: table.IsIncremental,
309+
TablePKConstraint: table.PkConstraintName,
300310
}
301311
metadata := NewSchemaMetadataFromOptions(opts)
302312
return arrow.NewSchema(fields, &metadata)

‎types/extensions_test.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package types
2+
3+
import (
4+
"testing"
5+
6+
"github.com/apache/arrow/go/v12/arrow"
7+
"github.com/apache/arrow/go/v12/arrow/array"
8+
"github.com/apache/arrow/go/v12/arrow/memory"
9+
"github.com/google/uuid"
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestValueStrRoundTrip(t *testing.T) {
15+
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
16+
defer mem.AssertSize(t, 0)
17+
18+
cases := []struct {
19+
arr arrow.Array
20+
builder array.Builder
21+
}{
22+
{
23+
arr: func() arrow.Array {
24+
b := NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType()))
25+
defer b.Release()
26+
27+
b.AppendNull()
28+
b.Append(mustParseInet("192.168.0.0/24"))
29+
b.AppendNull()
30+
b.Append(mustParseInet("192.168.0.0/25"))
31+
b.AppendNull()
32+
33+
return b.NewInetArray()
34+
}(),
35+
builder: NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())),
36+
},
37+
{
38+
arr: func() arrow.Array {
39+
b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType()))
40+
defer b.Release()
41+
42+
b.AppendNull()
43+
b.Append(map[string]any{"a": 1, "b": 2})
44+
b.AppendNull()
45+
b.Append([]any{1, 2, 3})
46+
b.AppendNull()
47+
b.Append(map[string]any{"MyKey": "A\u0026B"})
48+
b.AppendNull()
49+
50+
return b.NewJSONArray()
51+
}(),
52+
builder: NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())),
53+
},
54+
{
55+
arr: func() arrow.Array {
56+
b := NewMacBuilder(array.NewExtensionBuilder(mem, NewMacType()))
57+
defer b.Release()
58+
59+
b.AppendNull()
60+
b.Append(mustParseMac("00:00:00:00:00:01"))
61+
b.AppendNull()
62+
b.Append(mustParseMac("00:00:00:00:00:02"))
63+
b.AppendNull()
64+
65+
return b.NewMacArray()
66+
}(),
67+
builder: NewMacBuilder(array.NewExtensionBuilder(mem, NewMacType())),
68+
},
69+
{
70+
arr: func() arrow.Array {
71+
b := NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType()))
72+
defer b.Release()
73+
74+
b.AppendNull()
75+
b.Append(uuid.NameSpaceURL)
76+
b.AppendNull()
77+
b.Append(uuid.NameSpaceDNS)
78+
b.AppendNull()
79+
80+
return b.NewUUIDArray()
81+
}(),
82+
builder: NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())),
83+
},
84+
}
85+
86+
for _, tc := range cases {
87+
t.Run(tc.arr.DataType().(arrow.ExtensionType).ExtensionName(), func(t *testing.T) {
88+
defer tc.arr.Release()
89+
defer tc.builder.Release()
90+
t.Helper()
91+
92+
for i := 0; i < tc.arr.Len(); i++ {
93+
assert.NoError(t, tc.builder.AppendValueFromString(tc.arr.ValueStr(i)))
94+
}
95+
96+
arr := tc.builder.NewArray()
97+
defer arr.Release()
98+
99+
require.True(t, array.Equal(tc.arr, arr))
100+
})
101+
}
102+
}

‎types/inet.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,13 @@ func (b *InetBuilder) UnsafeAppend(v *net.IPNet) {
3333
}
3434

3535
func (b *InetBuilder) AppendValues(v []*net.IPNet, valid []bool) {
36+
if len(v) != len(valid) && len(valid) != 0 {
37+
panic("len(v) != len(valid) && len(valid) != 0")
38+
}
39+
3640
data := make([]string, len(v))
3741
for i, v := range v {
38-
if !valid[i] {
42+
if len(valid) > 0 && !valid[i] {
3943
continue
4044
}
4145
data[i] = v.String()
@@ -113,6 +117,10 @@ func (b *InetBuilder) UnmarshalJSON(data []byte) error {
113117
return b.Unmarshal(dec)
114118
}
115119

120+
func (b *InetBuilder) NewInetArray() *InetArray {
121+
return b.NewExtensionArray().(*InetArray)
122+
}
123+
116124
// InetArray is a simple array which is a FixedSizeBinary(16)
117125
type InetArray struct {
118126
array.ExtensionArrayBase

‎types/inet_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func TestInetBuilder(t *testing.T) {
3535
mustParseInet("192.168.0.0/26"),
3636
mustParseInet("192.168.0.0/27"),
3737
}
38-
b.AppendValues(values, []bool{true, true})
38+
b.AppendValues(values, nil)
3939

4040
require.Equal(t, 6, b.Len(), "unexpected Len()")
4141

‎types/json.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,14 @@ func (b *JSONBuilder) AppendValueFromString(s string) error {
5454
}
5555

5656
func (b *JSONBuilder) AppendValues(v []any, valid []bool) {
57+
if len(v) != len(valid) && len(valid) != 0 {
58+
panic("len(v) != len(valid) && len(valid) != 0")
59+
}
60+
5761
data := make([][]byte, len(v))
5862
var err error
5963
for i := range v {
60-
if !valid[i] {
64+
if len(valid) > 0 && !valid[i] {
6165
continue
6266
}
6367
// per https://github.com/cloudquery/plugin-sdk/issues/622
@@ -106,6 +110,10 @@ func (b *JSONBuilder) UnmarshalJSON(data []byte) error {
106110
return b.Unmarshal(dec)
107111
}
108112

113+
func (b *JSONBuilder) NewJSONArray() *JSONArray {
114+
return b.NewExtensionArray().(*JSONArray)
115+
}
116+
109117
// JSONArray is a simple array which is a Binary
110118
type JSONArray struct {
111119
array.ExtensionArrayBase

‎types/json_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestJSONBuilder(t *testing.T) {
2828
map[string]any{"e": 5, "f": 6},
2929
map[string]any{"g": 7, "h": 8},
3030
}
31-
b.AppendValues(values, []bool{true, true})
31+
b.AppendValues(values, nil)
3232

3333
require.Equal(t, 6, b.Len(), "unexpected Len()")
3434

@@ -172,7 +172,7 @@ func TestJSONArray_GetOneForMarshal(t *testing.T) {
172172
}
173173
}
174174

175-
func TestJSONArray_ValueStr(t *testing.T) {
175+
func TestJSONArray_ValueStrParse(t *testing.T) {
176176
cases := []struct {
177177
name string
178178
data string

‎types/mac.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,13 @@ func (b *MacBuilder) UnsafeAppend(v net.HardwareAddr) {
2929
}
3030

3131
func (b *MacBuilder) AppendValues(v []net.HardwareAddr, valid []bool) {
32+
if len(v) != len(valid) && len(valid) != 0 {
33+
panic("len(v) != len(valid) && len(valid) != 0")
34+
}
35+
3236
data := make([][]byte, len(v))
3337
for i, v := range v {
34-
if !valid[i] {
38+
if len(valid) > 0 && !valid[i] {
3539
continue
3640
}
3741
data[i] = v
@@ -107,6 +111,10 @@ func (b *MacBuilder) UnmarshalJSON(data []byte) error {
107111
return b.Unmarshal(dec)
108112
}
109113

114+
func (b *MacBuilder) NewMacArray() *MacArray {
115+
return b.NewExtensionArray().(*MacArray)
116+
}
117+
110118
// MacArray is a simple array which is a wrapper around a BinaryArray
111119
type MacArray struct {
112120
array.ExtensionArrayBase

‎types/mac_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func TestMacBuilder(t *testing.T) {
3535
mustParseMac("00:00:00:00:00:03"),
3636
mustParseMac("00:00:00:00:00:04"),
3737
}
38-
b.AppendValues(values, []bool{true, true})
38+
b.AppendValues(values, nil)
3939

4040
require.Equal(t, 6, b.Len(), "unexpected Len()")
4141

0 commit comments

Comments
 (0)