Skip to content

Commit 715bfe8

Browse files
candiduslynxcandiduslynx
authored
fix: Update engine.parameters validation (#14719)
Closes #14616
1 parent d1d6bdc commit 715bfe8

7 files changed

Lines changed: 58 additions & 20 deletions

File tree

‎plugins/destination/clickhouse/client/spec/engine.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,15 @@ const (
1111
MergeTree = "MergeTree"
1212
)
1313

14+
// Engine allows to specify a custom table engine to be used.
1415
type Engine struct {
15-
Name string `json:"name,omitempty" jsonschema:"pattern=^.*MergeTree$,default=MergeTree"`
16-
Parameters []any `json:"parameters,omitempty"`
16+
// Name of the table engine.
17+
// Only [`*MergeTree` family](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family) is supported at the moment.
18+
Name string `json:"name,omitempty" jsonschema:"pattern=^.*MergeTree$,default=MergeTree"`
19+
20+
// Engine parameters.
21+
// Currently, no restrictions are imposed on the parameter types.
22+
Parameters []any `json:"parameters,omitempty"`
1723
}
1824

1925
func (e *Engine) String() string {
@@ -46,7 +52,7 @@ func (e *Engine) params() []string {
4652
res[i] = "false"
4753
}
4854
default:
49-
panic(fmt.Errorf("unsupported engine option type %T", t))
55+
res[i] = fmt.Sprint(p)
5056
}
5157
}
5258

‎plugins/destination/clickhouse/client/spec/gen/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212

1313
func main() {
1414
fmt.Println("Generating JSON schema for plugin spec")
15-
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"))
15+
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"),
16+
jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/destination/clickhouse/client/spec", path.Join(currDir(), "..")),
17+
)
1618
}
1719

1820
func currDir() string {

‎plugins/destination/clickhouse/client/spec/schema.json‎

Lines changed: 18 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/destination/clickhouse/client/spec/spec.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,35 @@ import (
1111
"github.com/invopop/jsonschema"
1212
)
1313

14+
// CloudQuery ClickHouse destination plugin spec.
1415
type Spec struct {
16+
// Connection string to connect to the database.
17+
// See [SDK documentation](https://github.com/ClickHouse/clickhouse-go#dsn) for more details.
1518
ConnectionString string `json:"connection_string,omitempty" jsonschema:"required,minLength=1"`
16-
Cluster string `json:"cluster,omitempty"`
1719

20+
// Cluster name to be used for [distributed DDL](https://clickhouse.com/docs/en/sql-reference/distributed-ddl).
21+
// If the value is empty, DDL operations will affect only the server the plugin is connected to.
22+
Cluster string `json:"cluster,omitempty"`
23+
24+
// Engine to be used for tables.
25+
// Only [`*MergeTree` family](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family) is supported at the moment.
1826
Engine *Engine `json:"engine,omitempty"`
1927

28+
// PEM-encoded certificate authorities.
29+
// When set, a certificate pool will be created by appending the certificates to the system pool.
30+
//
31+
// See [file variable substitution](/docs/advanced-topics/environment-variable-substitution#file-variable-substitution-example)
32+
// for how to read this value from a file.
2033
CACert string `json:"ca_cert,omitempty"`
2134

22-
BatchSize int `json:"batch_size,omitempty" jsonschema:"minimum=1,default=10000"`
23-
BatchSizeBytes int `json:"batch_size_bytes,omitempty" jsonschema:"minimum=1,default=5242880"`
24-
BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`
35+
// This parameter controls the maximum amount of items may be grouped together to be written as a single write.
36+
BatchSize int `json:"batch_size,omitempty" jsonschema:"minimum=1,default=10000"`
37+
38+
// This parameter controls the maximum size of items that may be grouped together to be written as a single write.
39+
BatchSizeBytes int `json:"batch_size_bytes,omitempty" jsonschema:"minimum=1,default=5242880"`
40+
41+
// This parameter controls the maximum interval between batch writes.
42+
BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`
2543
}
2644

2745
func (s *Spec) Options() (*clickhouse.Options, error) {

‎plugins/destination/clickhouse/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,4 +137,4 @@ require (
137137
replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c
138138

139139
// github.com/cloudquery/jsonschema @ cqmain
140-
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231012111802-b28735982a93
140+
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f

‎plugins/destination/clickhouse/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,8 @@ github.com/cloudquery/cloudquery-api-go v1.4.1 h1:ag//nY7xWo+7LiBmS7FX6OSSIrutgT
9494
github.com/cloudquery/cloudquery-api-go v1.4.1/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
9595
github.com/cloudquery/codegen v0.3.9 h1:l3Pdq8PNVQ56P6VkUkCkqRm7CfDWASA2HL0oJXMnwnY=
9696
github.com/cloudquery/codegen v0.3.9/go.mod h1:c5+13oL0GRo42AEfFZey9k5TZpwSFdwBvtxXPFC8W/I=
97-
github.com/cloudquery/jsonschema v0.0.0-20231012111802-b28735982a93 h1:Rgtj0YMsk5BGD76Y38xCAHEtfOguxntnJO/q+oCAry4=
98-
github.com/cloudquery/jsonschema v0.0.0-20231012111802-b28735982a93/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
97+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
98+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
9999
github.com/cloudquery/plugin-pb-go v1.12.3 h1:rLK3/RR70/BX8tj2QzTnrjkxQhzfAT7SXEEEGuKr7Rk=
100100
github.com/cloudquery/plugin-pb-go v1.12.3/go.mod h1:CYorX3zCHF9ByoOgdBOuwLX/2vVCDH6/FoREOE3oH+w=
101101
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=

‎website/pages/docs/plugins/destinations/clickhouse/overview.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ This is the (nested) spec used by the ClickHouse destination plugin.
4343

4444
PEM-encoded certificate authorities.
4545
When set, a certificate pool will be created by appending the certificates to the system pool.
46+
4647
See [file variable substitution](/docs/advanced-topics/environment-variable-substitution#file-variable-substitution-example)
4748
for how to read this value from a file.
4849

@@ -61,7 +62,7 @@ This is the (nested) spec used by the ClickHouse destination plugin.
6162

6263
- `batch_timeout` (`duration`) (optional) (default: `20s`)
6364

64-
This parameter controls the timeout for writing a single batch.
65+
This parameter controls the maximum interval between batch writes.
6566

6667
#### ClickHouse table engine
6768

@@ -75,7 +76,7 @@ This option allows to specify a custom table engine to be used.
7576
- `parameters` (array of parameters) (optional) (default: empty)
7677

7778
Engine parameters.
78-
Parameters can be strings, numbers, booleans.
79+
Currently, no restrictions are imposed on the parameter types.
7980

8081
```yaml copy
8182
kind: destination

0 commit comments

Comments
 (0)