Skip to content

Commit 9bd1185

Browse files
candiduslynxcandiduslynx
authored
feat: Add JSON schema for plugin spec (#14623)
Closes #14616
1 parent 1464f39 commit 9bd1185

16 files changed

Lines changed: 501 additions & 78 deletions

File tree

‎.github/workflows/dest_clickhouse.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,12 @@ jobs:
6464
args: "--config ../../.golangci.yml"
6565
skip-pkg-cache: true
6666
skip-build-cache: true
67-
- run: go mod tidy
67+
- name: gen
68+
if: github.event_name == 'pull_request'
69+
run: make gen
70+
- name: Fail if generation updated files
71+
if: github.event_name == 'pull_request'
72+
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
6873
- name: Build
6974
run: go build .
7075
- name: Test ClickHouse

‎plugins/destination/clickhouse/Makefile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,11 @@ test:
77
.PHONY: lint
88
lint:
99
golangci-lint run --config ../../.golangci.yml
10+
11+
.PHONY: gen-spec-schema
12+
gen-spec-schema:
13+
go run client/spec/gen/main.go
14+
15+
# All gen targets
16+
.PHONY: gen
17+
gen: gen-spec-schema

‎plugins/destination/clickhouse/client/client.go‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/ClickHouse/clickhouse-go/v2"
88
"github.com/ClickHouse/clickhouse-go/v2/lib/proto"
9+
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/client/spec"
910
"github.com/cloudquery/plugin-sdk/v4/message"
1011
"github.com/cloudquery/plugin-sdk/v4/plugin"
1112
"github.com/cloudquery/plugin-sdk/v4/writers/batchwriter"
@@ -16,7 +17,7 @@ import (
1617
type Client struct {
1718
conn clickhouse.Conn
1819
database string
19-
spec *Spec
20+
spec *spec.Spec
2021

2122
logger zerolog.Logger
2223
writer *batchwriter.BatchWriter
@@ -40,24 +41,24 @@ func (c *Client) Close(ctx context.Context) error {
4041
}
4142

4243
func New(_ context.Context, logger zerolog.Logger, specBytes []byte, _ plugin.NewClientOptions) (plugin.Client, error) {
43-
var spec Spec
44-
if err := json.Unmarshal(specBytes, &spec); err != nil {
44+
var s spec.Spec
45+
if err := json.Unmarshal(specBytes, &s); err != nil {
4546
return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
4647
}
47-
spec.SetDefaults()
48-
if err := spec.Validate(); err != nil {
48+
s.SetDefaults()
49+
if err := s.Validate(); err != nil {
4950
return nil, err
5051
}
5152

52-
options, err := spec.Options()
53+
options, err := s.Options()
5354
if err != nil {
5455
return nil, fmt.Errorf("failed to prepare connect options %w", err)
5556
}
5657

5758
l := logger.With().
5859
Str("module", "dest-clickhouse").
5960
Str("database", options.Auth.Database).
60-
Str("cluster", spec.Cluster).
61+
Str("cluster", s.Cluster).
6162
Logger()
6263
options.Debugf = l.Printf
6364

@@ -80,14 +81,14 @@ func New(_ context.Context, logger zerolog.Logger, specBytes []byte, _ plugin.Ne
8081
c := &Client{
8182
conn: conn,
8283
database: options.Auth.Database,
83-
spec: &spec,
84+
spec: &s,
8485
logger: l,
8586
}
8687
c.writer, err = batchwriter.New(c,
8788
batchwriter.WithLogger(l),
88-
batchwriter.WithBatchSize(spec.BatchSize),
89-
batchwriter.WithBatchSizeBytes(spec.BatchSizeBytes),
90-
batchwriter.WithBatchTimeout(spec.BatchTimeout.Duration()),
89+
batchwriter.WithBatchSize(s.BatchSize),
90+
batchwriter.WithBatchSizeBytes(s.BatchSizeBytes),
91+
batchwriter.WithBatchTimeout(s.BatchTimeout.Duration()),
9192
)
9293
if err != nil {
9394
return nil, err

‎plugins/destination/clickhouse/client/client_test.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import (
66
"os"
77
"testing"
88

9+
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/client/spec"
10+
internalPlugin "github.com/cloudquery/cloudquery/plugins/destination/clickhouse/resources/plugin"
911
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/typeconv/ch/types"
1012
"github.com/cloudquery/plugin-sdk/v4/plugin"
1113
"github.com/goccy/go-json"
@@ -26,8 +28,12 @@ func getTestConnection() string {
2628

2729
func TestPlugin(t *testing.T) {
2830
ctx := context.Background()
29-
p := plugin.NewPlugin("clickhouse", "development", New)
30-
s := &Spec{ConnectionString: getTestConnection()}
31+
p := plugin.NewPlugin("clickhouse",
32+
internalPlugin.Version,
33+
New,
34+
plugin.WithJSONSchema(spec.JSONSchema),
35+
)
36+
s := &spec.Spec{ConnectionString: getTestConnection()}
3137
b, err := json.Marshal(s)
3238
require.NoError(t, err)
3339
require.NoError(t, p.Init(ctx, b, plugin.NewClientOptions{}))

‎plugins/destination/clickhouse/client/migrate.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,15 @@ package client
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
6-
77
"slices"
88

99
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/queries"
1010
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/typeconv"
1111
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/util"
1212
"github.com/cloudquery/plugin-sdk/v4/message"
1313
"github.com/cloudquery/plugin-sdk/v4/schema"
14-
"github.com/pkg/errors"
1514
"golang.org/x/sync/errgroup"
1615
)
1716

plugins/destination/clickhouse/queries/engine.go renamed to plugins/destination/clickhouse/client/spec/engine.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package queries
1+
package spec
22

33
import (
44
"encoding/json"
@@ -12,7 +12,7 @@ const (
1212
)
1313

1414
type Engine struct {
15-
Name string `json:"name,omitempty"`
15+
Name string `json:"name,omitempty" jsonschema:"pattern=^.*MergeTree$,default=MergeTree"`
1616
Parameters []any `json:"parameters,omitempty"`
1717
}
1818

‎plugins/destination/clickhouse/client/spec/engine_test.go‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package spec
2+
3+
import (
4+
"testing"
5+
6+
"github.com/cloudquery/codegen/jsonschema"
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestEngineJSONSchema(t *testing.T) {
11+
schema, err := jsonschema.Generate(Engine{})
12+
require.NoError(t, err)
13+
14+
jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
15+
{
16+
Name: "null",
17+
Err: true,
18+
Spec: `null`,
19+
},
20+
{
21+
Name: "bad",
22+
Err: true,
23+
Spec: `123`,
24+
},
25+
{
26+
Name: "empty",
27+
Spec: `{}`,
28+
},
29+
{
30+
Name: "extra keyword",
31+
Err: true,
32+
Spec: `{"extra":true}`,
33+
},
34+
{
35+
Name: "empty name",
36+
Err: true,
37+
Spec: `{"name":""}`,
38+
},
39+
{
40+
Name: "null name",
41+
Err: true,
42+
Spec: `{"name":null}`,
43+
},
44+
{
45+
Name: "bad name type",
46+
Err: true,
47+
Spec: `{"name":123}`,
48+
},
49+
{
50+
Name: "bad name format",
51+
Err: true,
52+
Spec: `{"name":"SomeEngine"}`,
53+
},
54+
{
55+
Name: "name: MergeTree",
56+
Spec: `{"name":"MergeTree"}`,
57+
},
58+
{
59+
Name: "name: SomeRubbishMergeTree",
60+
Spec: `{"name":"SomeRubbishMergeTree"}`,
61+
},
62+
{
63+
Name: "empty parameters",
64+
Spec: `{"parameters":[]}`,
65+
},
66+
{
67+
Name: "null parameters",
68+
Spec: `{"parameters":null}`,
69+
},
70+
{
71+
Name: "bad parameters",
72+
Err: true, // it's either null or array
73+
Spec: `{"parameters":123}`,
74+
},
75+
{
76+
Name: "parameters empty string entry",
77+
Spec: `{"parameters":[""]}`,
78+
},
79+
{
80+
Name: "parameters string entry",
81+
Spec: `{"parameters":["123"]}`,
82+
},
83+
{
84+
Name: "parameters zero float entry",
85+
Spec: `{"parameters":[0.0]}`,
86+
},
87+
{
88+
Name: "parameters positive float entry",
89+
Spec: `{"parameters":[5.7]}`,
90+
},
91+
{
92+
Name: "parameters negative float entry",
93+
Spec: `{"parameters":[-5.7]}`,
94+
},
95+
{
96+
Name: "parameters zero integer entry",
97+
Spec: `{"parameters":[0]}`,
98+
},
99+
{
100+
Name: "parameters positive integer entry",
101+
Spec: `{"parameters":[123]}`,
102+
},
103+
{
104+
Name: "parameters negative integer entry",
105+
Spec: `{"parameters":[-123]}`,
106+
},
107+
{
108+
Name: "parameters empty object entry",
109+
Spec: `{"parameters":[{}]}`,
110+
},
111+
{
112+
Name: "parameters non-empty object entry",
113+
Spec: `{"parameters":[{"a":[{"b":123}]}]}`,
114+
},
115+
{
116+
Name: "parameters empty array entry",
117+
Spec: `{"parameters":[[]]}`,
118+
},
119+
{
120+
Name: "parameters non-empty array entry",
121+
Spec: `{"parameters":[[{"a":[{"b":123}]}]]}`,
122+
},
123+
{
124+
Name: "null parameters entry",
125+
Spec: `{"parameters":[null]}`,
126+
},
127+
})
128+
}

‎plugins/destination/clickhouse/client/spec/gen/main.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/client/spec"
10+
"github.com/cloudquery/codegen/jsonschema"
11+
)
12+
13+
func main() {
14+
fmt.Println("Generating JSON schema for plugin spec")
15+
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"))
16+
}
17+
18+
func currDir() string {
19+
_, filename, _, ok := runtime.Caller(0)
20+
if !ok {
21+
log.Fatal("Failed to get caller information")
22+
}
23+
return path.Dir(filename)
24+
}

‎plugins/destination/clickhouse/client/spec/schema.json‎

Lines changed: 84 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)