Skip to content

Commit ad3c772

Browse files
yevgenypatsyevgenypats
authored
feat(azure)!: Update to SDK V3 Arrow native (#10974)
Closes #10735 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10974) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 7c0e8db commit ad3c772

966 files changed

Lines changed: 4585 additions & 4576 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/source_azure.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ jobs:
144144
- name: Setup CloudQuery
145145
uses: cloudquery/setup-cloudquery@v3
146146
with:
147-
version: 'v2.5.3'
147+
version: 'v3.5.0'
148148
- name: Migrate DB
149149
run: cloudquery migrate test/policy_cq_config.yml
150150
env:

‎plugins/source/azure/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ import (
1919
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
2020
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription"
2121
"github.com/cloudquery/plugin-pb-go/specs"
22-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
23-
"github.com/cloudquery/plugin-sdk/v2/schema"
22+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
23+
"github.com/cloudquery/plugin-sdk/v3/schema"
2424
"github.com/rs/zerolog"
2525
"github.com/thoas/go-funk"
2626
"golang.org/x/exp/maps"

‎plugins/source/azure/client/columns.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,20 @@
11
package client
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/v2/schema"
4+
"github.com/apache/arrow/go/v13/arrow"
5+
"github.com/cloudquery/plugin-sdk/v3/schema"
56
)
67

78
var (
89
SubscriptionID = schema.Column{
910
Name: "subscription_id",
10-
Type: schema.TypeString,
11+
Type: arrow.BinaryTypes.String,
1112
Resolver: ResolveAzureSubscription,
1213
}
1314
SubscriptionIDPK = schema.Column{
14-
Name: "subscription_id",
15-
Type: schema.TypeString,
16-
Resolver: ResolveAzureSubscription,
17-
CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true},
15+
Name: "subscription_id",
16+
Type: arrow.BinaryTypes.String,
17+
Resolver: ResolveAzureSubscription,
18+
PrimaryKey: true,
1819
}
1920
)

‎plugins/source/azure/client/multiplexer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"strings"
55

66
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/billing/armbilling"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
)
99

1010
func SubscriptionMultiplexRegisteredNamespace(table, namespace string) func(schema.ClientMeta) []schema.ClientMeta {

‎plugins/source/azure/client/resolvers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"context"
55

6-
"github.com/cloudquery/plugin-sdk/v2/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/schema"
77
)
88

99
func ResolveAzureSubscription(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {

‎plugins/source/azure/client/testing.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ import (
1919
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/billing/armbilling"
2020
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
2121
"github.com/cloudquery/plugin-pb-go/specs"
22-
"github.com/cloudquery/plugin-sdk/v2/faker"
23-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
24-
"github.com/cloudquery/plugin-sdk/v2/schema"
22+
"github.com/cloudquery/plugin-sdk/v3/faker"
23+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
24+
"github.com/cloudquery/plugin-sdk/v3/schema"
2525
"github.com/rs/zerolog"
2626
)
2727

‎plugins/source/azure/client/transformer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"reflect"
55

6-
"github.com/cloudquery/plugin-sdk/v2/transformers"
6+
"github.com/cloudquery/plugin-sdk/v3/transformers"
77
)
88

99
var _ transformers.NameTransformer = ETagNameTransformer

‎plugins/source/azure/go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,9 @@ require (
9898
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/windowsiot/armwindowsiot v1.0.0
9999
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/workloads/armworkloads v0.2.0
100100
github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v0.1.0
101+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
101102
github.com/cloudquery/plugin-pb-go v1.0.8
102-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
103+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
103104
github.com/gorilla/mux v1.8.0
104105
github.com/mitchellh/hashstructure/v2 v2.0.2
105106
github.com/rs/zerolog v1.29.0
@@ -116,8 +117,8 @@ require (
116117
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
117118
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
118119
github.com/andybalholm/brotli v1.0.5 // indirect
119-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
120120
github.com/apache/thrift v0.16.0 // indirect
121+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
121122
github.com/davecgh/go-spew v1.1.1 // indirect
122123
github.com/getsentry/sentry-go v0.20.0 // indirect
123124
github.com/ghodss/yaml v1.0.0 // indirect
@@ -138,6 +139,7 @@ require (
138139
github.com/mattn/go-isatty v0.0.18 // indirect
139140
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
140141
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
142+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
141143
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
142144
github.com/pmezard/go-difflib v1.0.0 // indirect
143145
github.com/spf13/cast v1.5.0 // indirect

‎plugins/source/azure/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
247247
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
248248
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
249249
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
250+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
251+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
250252
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
251253
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
252254
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -380,6 +382,7 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4
380382
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
381383
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
382384
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
385+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
383386
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
384387
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
385388
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=

‎plugins/source/azure/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/azure/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://be7c45692567444299f8bef3de545b86@o1396617.ingest.sentry.io/6747596"

0 commit comments

Comments
 (0)