Skip to content

Commit a35b8e8

Browse files
yevgenypatsyevgenypats
authored
feat: Update all plugins to SDK with metrics and DFS scheduler (#3286)
Should be merged after this SDK PR cloudquery/plugin-sdk#318
1 parent 3fb1fd4 commit a35b8e8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+146
-258
lines changed

‎.gitignore‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ test_init_config.yml
2222
telemetry-random-id
2323
go.work
2424
go.work.sum
25+
go.work.backup
26+
go.work.sum.backup
2527
cli/examples
2628
cli/cli
27-
plugins/source/aws/aws
28-
plugins/source/gcp/gcp

‎plugins/destination/postgresql/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ test:
66

77
.PHONY: lint
88
lint:
9-
@golangci-lint run --timeout 10m --verbose
9+
@golangci-lint run --config ../../.golangci.yml --timeout 10m --verbose

‎plugins/destination/postgresql/client/client.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ type Client struct {
2424
pgType pgType
2525
batchSize int
2626
batch *pgx.Batch
27+
metrics plugins.DestinationMetrics
2728
}
2829

2930
type pgTablePrimaryKeys struct {
@@ -116,6 +117,10 @@ func New(ctx context.Context, logger zerolog.Logger, spec specs.Destination) (pl
116117
return c, nil
117118
}
118119

120+
func (c *Client) Metrics() plugins.DestinationMetrics {
121+
return c.metrics
122+
}
123+
119124
func (c *Client) Close(ctx context.Context) error {
120125
var err error
121126
if c.conn == nil {

‎plugins/destination/postgresql/go.mod‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/cloudquery/cloudquery/plugins/destination/postgresql
33
go 1.19
44

55
require (
6-
github.com/cloudquery/plugin-sdk v0.13.14
6+
github.com/cloudquery/plugin-sdk v0.13.15
77
github.com/google/go-cmp v0.5.9
88
github.com/google/uuid v1.3.0
99
github.com/jackc/pgtype v1.12.0
@@ -37,9 +37,6 @@ require (
3737
github.com/spf13/pflag v1.0.5 // indirect
3838
github.com/stretchr/testify v1.8.0 // indirect
3939
github.com/thoas/go-funk v0.9.3-0.20221027085339-5573bc209e28 // indirect
40-
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
41-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
42-
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
4340
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
4441
golang.org/x/net v0.0.0-20220920203100-d0c6ba3f52d9 // indirect
4542
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 // indirect

‎plugins/destination/postgresql/go.sum‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
33
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
44
github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
55
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
6-
github.com/cloudquery/plugin-sdk v0.13.14 h1:GC7dv4F8IMFWvpUq0sOZI2NmAtsJSKu3Ztu18JGqXnU=
7-
github.com/cloudquery/plugin-sdk v0.13.14/go.mod h1:8mt4PDjek0NkldnWFnM/i0zpfyUYVcRy/imEN1L/Cxs=
6+
github.com/cloudquery/plugin-sdk v0.13.15 h1:ra2jUGslbCphIF3BrtlwUf8/pbMrp0FDuruE3l6/k1s=
7+
github.com/cloudquery/plugin-sdk v0.13.15/go.mod h1:8mt4PDjek0NkldnWFnM/i0zpfyUYVcRy/imEN1L/Cxs=
88
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
99
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
1010
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -176,13 +176,6 @@ github.com/thoas/go-funk v0.9.3-0.20221027085339-5573bc209e28 h1:xQnb8OgtW7FLvEt
176176
github.com/thoas/go-funk v0.9.3-0.20221027085339-5573bc209e28/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
177177
github.com/vgarvardt/pgx-google-uuid/v4 v4.0.0 h1:INIpwIh4qnQW4oI4afnmWDR2MYPK1IWYlDCxazNna/E=
178178
github.com/vgarvardt/pgx-google-uuid/v4 v4.0.0/go.mod h1:qWxaeeNYpTuoPNhQCOnXOyWY33AxxfTcNT+o/vnJLFM=
179-
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
180-
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
181-
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
182-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
183-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
184-
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
185-
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
186179
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
187180
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
188181
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

‎plugins/source/aws/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
aws

‎plugins/source/aws/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ gen-docs:
2626

2727
.PHONY: lint
2828
lint:
29-
golangci-lint run
29+
golangci-lint run --config ../../.golangci.yml
3030

3131
.PHONY: gen-code
3232
gen-code:

‎plugins/source/aws/client/client.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,8 +302,7 @@ func (c *Client) Logger() *zerolog.Logger {
302302
return &c.logger
303303
}
304304

305-
// Identify the given client
306-
func (c *Client) Identify() string {
305+
func (c *Client) ID() string {
307306
return strings.TrimRight(strings.Join([]string{
308307
c.AccountID,
309308
c.Region,

‎plugins/source/aws/client/multiplexers.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func AccountMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
3232
region := getRegion(client.ServicesManager.services[partition][accountID])
3333
// Ensure that the region is always set by a region that has been initialized
3434
if region == "" {
35-
meta.Logger().Trace().Str("accountID", accountID).Str("partition", partition).Msg("no valid regions have been specified for this account")
35+
meta.(*Client).Logger().Trace().Str("accountID", accountID).Str("partition", partition).Msg("no valid regions have been specified for this account")
3636
continue
3737
}
3838
l = append(l, client.withPartitionAccountIDAndRegion(partition, accountID, region))
@@ -49,7 +49,7 @@ func ServiceAccountRegionMultiplexer(service string) func(meta schema.ClientMeta
4949
for accountID := range client.ServicesManager.services[partition] {
5050
for region := range client.ServicesManager.services[partition][accountID] {
5151
if !isSupportedServiceForRegion(service, region) {
52-
meta.Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
52+
meta.(*Client).Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
5353
continue
5454
}
5555
l = append(l, client.withPartitionAccountIDAndRegion(partition, accountID, region))
@@ -68,7 +68,7 @@ func ServiceAccountRegionNamespaceMultiplexer(service string) func(meta schema.C
6868
for accountID := range client.ServicesManager.services[partition] {
6969
for region := range client.ServicesManager.services[partition][accountID] {
7070
if !isSupportedServiceForRegion(service, region) {
71-
meta.Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
71+
meta.(*Client).Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
7272
continue
7373
}
7474
for _, ns := range AllNamespaces {
@@ -91,7 +91,7 @@ func ServiceAccountRegionScopeMultiplexer(service string) func(meta schema.Clien
9191
l = append(l, client.withPartitionAccountIDRegionAndScope(partition, accountID, cloudfrontScopeRegion, wafv2types.ScopeCloudfront))
9292
for region := range client.ServicesManager.services[partition][accountID] {
9393
if !isSupportedServiceForRegion(service, region) {
94-
meta.Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
94+
meta.(*Client).Logger().Trace().Str("service", service).Str("region", region).Str("partition", partition).Msg("region is not supported for service")
9595
continue
9696
}
9797
l = append(l, client.withPartitionAccountIDRegionAndScope(partition, accountID, region, wafv2types.ScopeRegional))

‎plugins/source/aws/client/testing.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ func AwsMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.
4444
table,
4545
},
4646
newTestExecutionClient)
47-
plugins.TestSourcePluginSync(t, p, l, specs.Source{
47+
p.SetLogger(l)
48+
plugins.TestSourcePluginSync(t, p, specs.Source{
4849
Name: "dev",
4950
Version: version,
5051
Tables: []string{table.Name},

0 commit comments

Comments
 (0)