Skip to content

Commit 4426d4d

Browse files
candiduslynxcandiduslynx
authored
feat: Add JSON schema for plugin spec (#14558)
Closes #14553 Note [`uniqueItems`](https://json-schema.org/draft/2020-12/json-schema-validation#section-6.4.3) feature that's useful for the `subscriptions` & `skip_subscriptions` validation.
1 parent a665705 commit 4426d4d

File tree

12 files changed

+334
-40
lines changed

12 files changed

+334
-40
lines changed

‎plugins/source/azure/Makefile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ gen-docs: build
3131
lint:
3232
golangci-lint run --config ../../.golangci.yml
3333

34+
.PHONY: gen-spec-schema
35+
gen-spec-schema:
36+
go run client/spec/gen/main.go
37+
38+
# All gen targets
3439
.PHONY: gen
35-
gen: gen-docs
36-
echo "Done"
40+
gen: gen-spec-schema gen-docs

‎plugins/source/azure/client/client.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
2020
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions"
2121
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
22+
"github.com/cloudquery/cloudquery/plugins/source/azure/client/spec"
2223
"github.com/cloudquery/plugin-sdk/v4/schema"
2324
"github.com/rs/zerolog"
2425
"github.com/thoas/go-funk"
@@ -45,7 +46,7 @@ type Client struct {
4546
Creds azcore.TokenCredential
4647
Options *arm.ClientOptions
4748

48-
pluginSpec *Spec
49+
pluginSpec *spec.Spec
4950
BillingAccounts []*armbilling.Account
5051
BillingAccount *armbilling.Account
5152
BillingProfile *armbilling.Profile
@@ -236,7 +237,7 @@ func getCloudConfigFromSpec(specCloud string) (cloud.Configuration, error) {
236237
return cloud.Configuration{}, fmt.Errorf("unknown Azure cloud name %q. Supported values are %q", specCloud, maps.Keys(specCloudToConfig))
237238
}
238239

239-
func New(ctx context.Context, logger zerolog.Logger, s *Spec) (schema.ClientMeta, error) {
240+
func New(ctx context.Context, logger zerolog.Logger, s *spec.Spec) (schema.ClientMeta, error) {
240241
s.SetDefaults()
241242
uniqueSubscriptions := funk.Uniq(s.Subscriptions).([]string)
242243
c := &Client{

‎plugins/source/azure/client/spec.go‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎plugins/source/azure/client/spec/gen/main.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"path"
7+
"runtime"
8+
9+
"github.com/cloudquery/cloudquery/plugins/source/azure/client/spec"
10+
"github.com/cloudquery/codegen/jsonschema"
11+
)
12+
13+
func main() {
14+
fmt.Println("Generating JSON schema for plugin spec")
15+
jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json"))
16+
}
17+
18+
func currDir() string {
19+
_, filename, _, ok := runtime.Caller(0)
20+
if !ok {
21+
log.Fatal("Failed to get caller information")
22+
}
23+
return path.Dir(filename)
24+
}

‎plugins/source/azure/client/spec/schema.json‎

Lines changed: 64 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/azure/client/spec/spec.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package spec
2+
3+
import _ "embed"
4+
5+
type Spec struct {
6+
Subscriptions []string `json:"subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
7+
SkipSubscriptions []string `json:"skip_subscriptions" jsonschema:"minLength=1,uniqueItems=true"`
8+
CloudName string `json:"cloud_name" jsonschema:"minLength=1"`
9+
NormalizeIDs bool `json:"normalize_ids"`
10+
OIDCToken string `json:"oidc_token" jsonschema:"minLength=1"`
11+
Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"`
12+
DiscoveryConcurrency int `json:"discovery_concurrency" jsonschema:"minimum=1,default=400"`
13+
}
14+
15+
func (s *Spec) SetDefaults() {
16+
if s.DiscoveryConcurrency <= 0 {
17+
s.DiscoveryConcurrency = 400
18+
}
19+
if s.Concurrency <= 0 {
20+
const defaultConcurrency = 50000
21+
s.Concurrency = defaultConcurrency
22+
}
23+
}
24+
25+
//go:embed schema.json
26+
var JSONSchema string

‎plugins/source/azure/client/spec/spec_test.go‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package spec
2+
3+
import (
4+
"testing"
5+
6+
"github.com/cloudquery/codegen/jsonschema"
7+
)
8+
9+
func TestSpecJSONSchema(t *testing.T) {
10+
jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{
11+
{
12+
Name: "empty",
13+
Spec: `{}`,
14+
},
15+
{
16+
Name: "extra keyword",
17+
Err: true,
18+
Spec: `{"extra":true}`,
19+
},
20+
{
21+
Name: "empty subscriptions",
22+
Spec: `{"subscriptions":[]}`,
23+
},
24+
{
25+
Name: "null subscriptions",
26+
Spec: `{"subscriptions":null}`,
27+
},
28+
{
29+
Name: "bad subscriptions",
30+
Err: true,
31+
Spec: `{"subscriptions":123}`,
32+
},
33+
{
34+
Name: "empty subscriptions entry",
35+
Err: true,
36+
Spec: `{"subscriptions":[""]}`,
37+
},
38+
{
39+
Name: "null subscriptions entry",
40+
Err: true,
41+
Spec: `{"subscriptions":[null]}`,
42+
},
43+
{
44+
Name: "bad subscriptions entry",
45+
Err: true,
46+
Spec: `{"subscriptions":[123]}`,
47+
},
48+
{
49+
Name: "duplicate subscriptions entry",
50+
Err: true,
51+
Spec: `{"subscriptions":["a", "a"]}`,
52+
},
53+
{
54+
Name: "proper subscriptions entries",
55+
Spec: `{"subscriptions":["a", "b"]}`,
56+
},
57+
{
58+
Name: "empty skip_subscriptions",
59+
Spec: `{"skip_subscriptions":[]}`,
60+
},
61+
{
62+
Name: "null skip_subscriptions",
63+
Spec: `{"skip_subscriptions":null}`,
64+
},
65+
{
66+
Name: "bad skip_subscriptions",
67+
Err: true,
68+
Spec: `{"skip_subscriptions":123}`,
69+
},
70+
{
71+
Name: "empty skip_subscriptions entry",
72+
Err: true,
73+
Spec: `{"skip_subscriptions":[""]}`,
74+
},
75+
{
76+
Name: "null skip_subscriptions entry",
77+
Err: true,
78+
Spec: `{"skip_subscriptions":[null]}`,
79+
},
80+
{
81+
Name: "bad skip_subscriptions entry",
82+
Err: true,
83+
Spec: `{"skip_subscriptions":[123]}`,
84+
},
85+
{
86+
Name: "duplicate skip_subscriptions entry",
87+
Err: true,
88+
Spec: `{"skip_subscriptions":["a", "a"]}`,
89+
},
90+
{
91+
Name: "proper skip_subscriptions entries",
92+
Spec: `{"skip_subscriptions":["a", "b"]}`,
93+
},
94+
{
95+
Name: "empty cloud_name",
96+
Err: true,
97+
Spec: `{"cloud_name":""}`,
98+
},
99+
{
100+
Name: "null cloud_name",
101+
Err: true,
102+
Spec: `{"cloud_name":null}`,
103+
},
104+
{
105+
Name: "bad cloud_name",
106+
Err: true,
107+
Spec: `{"cloud_name":123}`,
108+
},
109+
{
110+
Name: "proper cloud_name",
111+
Spec: `{"cloud_name":"a"}`,
112+
},
113+
{
114+
Name: "normalize_ids:false",
115+
Spec: `{"normalize_ids":false}`,
116+
},
117+
{
118+
Name: "normalize_ids:true",
119+
Spec: `{"normalize_ids":true}`,
120+
},
121+
{
122+
Name: "null normalize_ids",
123+
Err: true,
124+
Spec: `{"normalize_ids":null}`,
125+
},
126+
{
127+
Name: "bad normalize_ids",
128+
Err: true,
129+
Spec: `{"normalize_ids":123}`,
130+
},
131+
{
132+
Name: "empty oidc_token",
133+
Err: true,
134+
Spec: `{"oidc_token":""}`,
135+
},
136+
{
137+
Name: "null oidc_token",
138+
Err: true,
139+
Spec: `{"oidc_token":null}`,
140+
},
141+
{
142+
Name: "bad oidc_token",
143+
Err: true,
144+
Spec: `{"oidc_token":123}`,
145+
},
146+
{
147+
Name: "proper oidc_token",
148+
Spec: `{"oidc_token":"a"}`,
149+
},
150+
{
151+
Name: "zero concurrency",
152+
Err: true,
153+
Spec: `{"concurrency":0}`,
154+
},
155+
{
156+
Name: "null concurrency",
157+
Err: true,
158+
Spec: `{"concurrency":null}`,
159+
},
160+
{
161+
Name: "bad concurrency",
162+
Err: true,
163+
Spec: `{"concurrency":"123"}`,
164+
},
165+
{
166+
Name: "proper concurrency",
167+
Spec: `{"concurrency":123}`,
168+
},
169+
{
170+
Name: "zero discovery_concurrency",
171+
Err: true,
172+
Spec: `{"discovery_concurrency":0}`,
173+
},
174+
{
175+
Name: "null discovery_concurrency",
176+
Err: true,
177+
Spec: `{"discovery_concurrency":null}`,
178+
},
179+
{
180+
Name: "bad discovery_concurrency",
181+
Err: true,
182+
Spec: `{"discovery_concurrency":"123"}`,
183+
},
184+
{
185+
Name: "proper discovery_concurrency",
186+
Spec: `{"discovery_concurrency":123}`,
187+
},
188+
})
189+
}

‎plugins/source/azure/client/testing.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111
"time"
1212

13+
"github.com/cloudquery/cloudquery/plugins/source/azure/client/spec"
1314
"github.com/cloudquery/plugin-sdk/v4/plugin"
1415
"github.com/cloudquery/plugin-sdk/v4/transformers"
1516
"github.com/gorilla/mux"
@@ -139,7 +140,7 @@ func MockTestHelper(t *testing.T, table *schema.Table, createServices func(*mux.
139140
TestSubscription: {&billingPeriod},
140141
},
141142
storageAccountKeys: &sync.Map{},
142-
pluginSpec: &Spec{
143+
pluginSpec: &spec.Spec{
143144
NormalizeIDs: true,
144145
},
145146
}

0 commit comments

Comments
 (0)