Skip to content

Commit 6cf28e4

Browse files
candiduslynxcandiduslynx
authored
fix: Properly handle cloud_name in JSON schema (cloudquery#14663)
1 parent 3050bb4 commit 6cf28e4

8 files changed

Lines changed: 113 additions & 45 deletions

File tree

‎plugins/source/azure/client/client.go‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
// Import all autorest modules
1212
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
1313
"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
14-
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
1514
"github.com/Azure/azure-sdk-for-go/sdk/azcore/log"
1615
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
1716
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
@@ -23,7 +22,6 @@ import (
2322
"github.com/cloudquery/plugin-sdk/v4/schema"
2423
"github.com/rs/zerolog"
2524
"github.com/thoas/go-funk"
26-
"golang.org/x/exp/maps"
2725
"golang.org/x/sync/errgroup"
2826
)
2927

@@ -223,20 +221,6 @@ func (c *Client) discoverResourceGroups(ctx context.Context) error {
223221
return errorGroup.Wait()
224222
}
225223

226-
func getCloudConfigFromSpec(specCloud string) (cloud.Configuration, error) {
227-
var specCloudToConfig = map[string]cloud.Configuration{
228-
"AzurePublic": cloud.AzurePublic,
229-
"AzureGovernment": cloud.AzureGovernment,
230-
"AzureChina": cloud.AzureChina,
231-
}
232-
233-
if v, ok := specCloudToConfig[specCloud]; ok {
234-
return v, nil
235-
}
236-
237-
return cloud.Configuration{}, fmt.Errorf("unknown Azure cloud name %q. Supported values are %q", specCloud, maps.Keys(specCloudToConfig))
238-
}
239-
240224
func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.ClientMeta, error) {
241225
s.SetDefaults()
242226
uniqueSubscriptions := funk.Uniq(s.Subscriptions).([]string)
@@ -248,7 +232,7 @@ func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.Clien
248232
}
249233

250234
if s.CloudName != "" {
251-
cloudConfig, err := getCloudConfigFromSpec(s.CloudName)
235+
cloudConfig, err := s.CloudConfig()
252236
if err != nil {
253237
return nil, err
254238
}

‎plugins/source/azure/client/spec/gen/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212

1313
func main() {
1414
fmt.Println("Generating JSON schema for plugin spec")
15-
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"))
15+
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"),
16+
jsonschema.WithAddGoComments("github.com/cloudquery/cloudquery/plugins/source/azure/client/spec", path.Join(currDir(), "..")),
17+
)
1618
}
1719

1820
func currDir() string {

‎plugins/source/azure/client/spec/schema.json‎

Lines changed: 17 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/azure/client/spec/spec.go‎

Lines changed: 63 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,65 @@
11
package spec
22

3-
import _ "embed"
3+
import (
4+
_ "embed"
5+
"fmt"
6+
7+
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
8+
"github.com/invopop/jsonschema"
9+
"golang.org/x/exp/maps"
10+
)
411

512
type Spec struct {
6-
Subscriptions []string `json:"subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
7-
SkipSubscriptions []string `json:"skip_subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
8-
CloudName string `json:"cloud_name" jsonschema:"minLength=1"`
9-
NormalizeIDs bool `json:"normalize_ids"`
10-
OIDCToken string `json:"oidc_token" jsonschema:"minLength=1"`
11-
Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"`
12-
DiscoveryConcurrency int `json:"discovery_concurrency" jsonschema:"minimum=1,default=400"`
13+
// Specify which subscriptions to sync data from.
14+
// Empty means all visible subscriptions.
15+
Subscriptions []string `json:"subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
16+
17+
// A list of subscription IDs that CloudQuery will skip syncing.
18+
// This is useful if CloudQuery is discovering the list of subscription IDs and there are some subscriptions that you want to not even attempt syncing.
19+
SkipSubscriptions []string `json:"skip_subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
20+
21+
// The name of the cloud environment to use.
22+
// See the [Azure CLI documentation](https://learn.microsoft.com/en-us/cli/azure/manage-clouds-azure-cli) for more information.
23+
CloudName string `json:"cloud_name" jsonschema:"minLength=1"`
24+
25+
// Enabling this setting will force all `id` column values to be lowercase.
26+
// This is useful to avoid case sensitivity and uniqueness issues around the `id` primary keys.
27+
NormalizeIDs bool `json:"normalize_ids"`
28+
29+
// An OIDC token can be used to authenticate with Azure instead of `AZURE_CLIENT_SECRET`.
30+
// This is useful for Azure AD workload identity federation.
31+
// When using this option, the `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` environment variables must be set.
32+
OIDCToken string `json:"oidc_token" jsonschema:"minLength=1"`
33+
34+
// The best effort maximum number of Go routines to use.
35+
// Lower this number to reduce memory usage.
36+
Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"`
37+
38+
// During initialization the Azure source plugin discovers all resource groups
39+
// and enabled resource providers per subscription, to be used later on during the sync process.
40+
// The plugin runs the discovery process in parallel.
41+
// This setting controls the maximum number of concurrent requests to the Azure API during discovery.
42+
// Only accounts with many subscriptions should require modifying this setting,
43+
// to either lower it to avoid network errors, or to increase it to speed up the discovery process.
44+
DiscoveryConcurrency int `json:"discovery_concurrency" jsonschema:"minimum=1,default=400"`
45+
}
46+
47+
var (
48+
specCloudToConfig = map[string]cloud.Configuration{
49+
"AzurePublic": cloud.AzurePublic,
50+
"AzureGovernment": cloud.AzureGovernment,
51+
"AzureChina": cloud.AzureChina,
52+
}
53+
// note: this should also be updated if new keys are added to specCloudToConfig
54+
specCloudToConfigKeys = []any{"AzurePublic", "AzureGovernment", "AzureChina"}
55+
)
56+
57+
func (s *Spec) CloudConfig() (cloud.Configuration, error) {
58+
if v, ok := specCloudToConfig[s.CloudName]; ok {
59+
return v, nil
60+
}
61+
62+
return cloud.Configuration{}, fmt.Errorf("unknown Azure cloud name %q. Supported values are %q", s.CloudName, maps.Keys(specCloudToConfig))
1363
}
1464

1565
func (s *Spec) SetDefaults() {
@@ -22,5 +72,10 @@ func (s *Spec) SetDefaults() {
2272
}
2373
}
2474

75+
func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) {
76+
cloudName := sc.Properties.Value("cloud_name")
77+
cloudName.Enum = append(cloudName.Enum, specCloudToConfigKeys...)
78+
}
79+
2580
//go:embed schema.json
2681
var JSONSchema string

‎plugins/source/azure/client/spec/spec_test.go‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"github.com/cloudquery/codegen/jsonschema"
77
)
88

9-
func TestSpecJSONSchema(t *testing.T) {
9+
func TestSpec_JSONSchemaExtend(t *testing.T) {
1010
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
1111
{
1212
Name: "empty",
@@ -107,8 +107,21 @@ func TestSpecJSONSchema(t *testing.T) {
107107
Spec: `{"cloud_name":123}`,
108108
},
109109
{
110-
Name: "proper cloud_name",
111-
Spec: `{"cloud_name":"a"}`,
110+
Name: "bad cloud_name value",
111+
Err: true,
112+
Spec: `{"cloud_name":"abc"}`,
113+
},
114+
{
115+
Name: "cloud_name: AzurePublic",
116+
Spec: `{"cloud_name":"AzurePublic"}`,
117+
},
118+
{
119+
Name: "cloud_name: AzureGovernment",
120+
Spec: `{"cloud_name":"AzureGovernment"}`,
121+
},
122+
{
123+
Name: "cloud_name: AzureChina",
124+
Spec: `{"cloud_name":"AzureChina"}`,
112125
},
113126
{
114127
Name: "normalize_ids:false",

‎plugins/source/azure/go.mod‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ require (
104104
github.com/cloudquery/plugin-sdk/v4 v4.15.2
105105
github.com/cockroachdb/cockroachdb-parser v0.0.0-20230705064001-302c9ad52e1a
106106
github.com/gorilla/mux v1.8.0
107+
github.com/invopop/jsonschema v0.11.0
107108
github.com/mitchellh/hashstructure/v2 v2.0.2
108109
github.com/mjibson/sqlfmt v0.5.0
109110
github.com/rs/zerolog v1.29.1
@@ -113,12 +114,6 @@ require (
113114
golang.org/x/sync v0.4.0
114115
)
115116

116-
// TODO: remove once all updates are merged
117-
replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c
118-
119-
// github.com/cloudquery/jsonschema @ cqmain
120-
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231013155745-f32a9237eda0
121-
122117
require (
123118
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
124119
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
@@ -177,7 +172,6 @@ require (
177172
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
178173
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
179174
github.com/inconshreveable/mousetrap v1.1.0 // indirect
180-
github.com/invopop/jsonschema v0.11.0 // indirect
181175
github.com/iris-contrib/schema v0.0.6 // indirect
182176
github.com/josharian/intern v1.0.0 // indirect
183177
github.com/json-iterator/go v1.1.12 // indirect
@@ -257,3 +251,9 @@ require (
257251
gopkg.in/yaml.v2 v2.4.0 // indirect
258252
gopkg.in/yaml.v3 v3.0.1 // indirect
259253
)
254+
255+
// TODO: remove once all updates are merged
256+
replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c
257+
258+
// github.com/cloudquery/jsonschema @ cqmain
259+
replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f

‎plugins/source/azure/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,8 @@ github.com/cloudquery/cloudquery-api-go v1.4.1 h1:ag//nY7xWo+7LiBmS7FX6OSSIrutgT
322322
github.com/cloudquery/cloudquery-api-go v1.4.1/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
323323
github.com/cloudquery/codegen v0.3.9 h1:l3Pdq8PNVQ56P6VkUkCkqRm7CfDWASA2HL0oJXMnwnY=
324324
github.com/cloudquery/codegen v0.3.9/go.mod h1:c5+13oL0GRo42AEfFZey9k5TZpwSFdwBvtxXPFC8W/I=
325-
github.com/cloudquery/jsonschema v0.0.0-20231013155745-f32a9237eda0 h1:4L/chcVQqiOQXC9Y9/s51mbX5qWwaKa5sGGNXHkkD/A=
326-
github.com/cloudquery/jsonschema v0.0.0-20231013155745-f32a9237eda0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
325+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
326+
github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
327327
github.com/cloudquery/plugin-pb-go v1.12.3 h1:rLK3/RR70/BX8tj2QzTnrjkxQhzfAT7SXEEEGuKr7Rk=
328328
github.com/cloudquery/plugin-pb-go v1.12.3/go.mod h1:CYorX3zCHF9ByoOgdBOuwLX/2vVCDH6/FoREOE3oH+w=
329329
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=

‎website/pages/docs/plugins/sources/azure/configuration.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ This is the (nested) spec used by the Azure source plugin.
2121

2222
- `concurrency` (`int`) (default: `50000`):
2323

24-
A best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
24+
The best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
2525

2626
- `discovery_concurrency` (`int`) (default: `400`)
2727

@@ -40,4 +40,6 @@ This is the (nested) spec used by the Azure source plugin.
4040

4141
- `oidc_token` (`string`) (default: empty)
4242

43-
An OIDC token can be used to authenticate with Azure instead of `AZURE_CLIENT_SECRET`. This is useful for Azure AD workload identity federation. When using this option, the `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` environment variables must be set.
43+
An OIDC token can be used to authenticate with Azure instead of `AZURE_CLIENT_SECRET`.
44+
This is useful for Azure AD workload identity federation.
45+
When using this option, the `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` environment variables must be set.

0 commit comments

Comments
 (0)