Skip to content

Commit a0a8f0a

Browse files
yevgenypatsyevgenypats
authored
feat: Move azure to avoid codegen. (#6276)
This shouldn't introduce any public facing changes. This should go together with: cloudquery/plugin-sdk#564 In general codegen creates a sub-optimal developer experience where you have to juggle between `codegen` and code to make any code-changes. Not only it creates a tough experience for the developer it also makes it hard to review and to contribute. This take here moves codegen only to the discovery phase and moves the transformation to be part of the SDK which will eventually deprecate the `codegen` package from the SDK completely and will avoid having two APIs which are almost identical but quite the same.
1 parent 8281faa commit a0a8f0a

868 files changed

Lines changed: 2202 additions & 15868 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎plugins/source/azure/Makefile‎

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
1-
# Generate mocks for mock/unit testing
2-
.PHONY: generate-mocks
3-
generate-mocks:
4-
go generate ./client/services/...
5-
61
# Test unit
72
.PHONY: test
83
test:
94
go test -race -timeout 3m ./...
105

11-
126
.PHONY: gen-docs
137
gen-docs:
148
rm -rf ./docs/tables/*
@@ -19,25 +13,6 @@ gen-docs:
1913
lint:
2014
golangci-lint run --config ../../.golangci.yml
2115

22-
.PHONY: codegen0
23-
codegen0:
24-
grep -rl '// Code generated by codegen0; DO NOT EDIT.' codegen1/recipes/* | xargs rm
25-
go run codegen0/main.go
26-
27-
.PHONY: codegen1
28-
codegen1:
29-
grep -rl '// Code generated by codegen1; DO NOT EDIT.' codegen2/recipes/* | xargs rm
30-
go run codegen1/main.go
31-
32-
.PHONY: codegen2
33-
codegen2:
34-
grep -rl '// Code generated by codegen2; DO NOT EDIT.' resources/* client/* | xargs rm
35-
go run codegen2/main.go
36-
3716
.PHONY: gen
38-
gen: codegen2 gen-docs
17+
gen: gen-docs
3918
echo "Done"
40-
41-
.PHONY: pregen
42-
pregen: codegen1 codegen2
43-
echo "Done"

‎plugins/source/azure/client/client.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,17 @@ const Namespacemicrosoft_keyvault = "microsoft.keyvault"
2121
const Namespacemicrosoft_cache = "microsoft.cache"
2222
const Namespacemicrosoft_search = "microsoft.search"
2323
const Namespacemicrosoft_logic = "microsoft.logic"
24+
const Namespacemicrosoft_costmanagement = "microsoft.costmanagement"
25+
const Namespacemicrosoft_storage = "microsoft.storage"
2426

2527
var otherNamespaces = []string{
2628
Namespacemicrosoft_dbformariadb,
2729
Namespacemicrosoft_keyvault,
2830
Namespacemicrosoft_cache,
2931
Namespacemicrosoft_search,
3032
Namespacemicrosoft_logic,
33+
Namespacemicrosoft_costmanagement,
34+
Namespacemicrosoft_storage,
3135
}
3236

3337
var allNamespaces = append(autoGeneratedNamespaces, otherNamespaces...)

‎plugins/source/azure/client/namespaces.go‎

Lines changed: 1 addition & 27 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/azure/codegen0/internal/azparser/discover.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ import (
88
)
99

1010
var packagesToSkip = map[string]bool{
11+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/costmanagement/armcostmanagement": true,
12+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/databoxedge/armdataboxedge": true,
13+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appplatform/armappplatform": true,
1114
// Manually generated recipes
1215
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault": true,
1316
// This is a special API and we create those recipes manually

‎plugins/source/azure/codegen0/internal/azparser/parser.go‎

Lines changed: 69 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,21 @@ var newGlobalFuncsToSkip = map[string]bool{
1919
// We are skipping operationsClient as this just list all operations available and it is quite static
2020
// so don't think it's of any use and we can always enable it later
2121
"NewOperationsClient": true,
22+
// This is mostly not needed and if it is needed we add this resource manually
23+
"NewSKUsClient": true,
2224
}
2325

2426
var newFuncToSkipPerPackage = map[string]map[string]bool{
27+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/synapse/armsynapse": {
28+
"NewKustoOperationsClient": true,
29+
},
2530
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/mariadb/armmariadb": {
2631
// we migrated this to manual written client as it has childs
2732
"NewServersClient": true,
2833
},
34+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/apimanagement/armapimanagement": {
35+
"NewSKUsClient": true,
36+
},
2937
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/mysql/armmysql": {
3038
"NewServersClient": true,
3139
},
@@ -36,6 +44,7 @@ var newFuncToSkipPerPackage = map[string]map[string]bool{
3644
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute": {
3745
// we migrated this to manual written client as it has childs
3846
"NewVirtualMachinesClient": true,
47+
"NewResourceSKUsClient": true,
3948
},
4049
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmos/armcosmos": {
4150
"NewDatabaseAccountsClient": true,
@@ -63,6 +72,8 @@ var newFuncToSkipPerPackage = map[string]map[string]bool{
6372
"NewApplyUpdatesClient": true,
6473
},
6574
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/security/armsecurity": {
75+
// ManualResource
76+
"NewSettingsClient": true,
6677
// Seems like a buggy resource that always returns a marshal error. maybe will be fixed in future Azure SDK
6778
"NewIngestionSettingsClient": true,
6879
// Seems like a buggy resource that always returns a marshal error. maybe will be fixed in future Azure SDK
@@ -115,11 +126,6 @@ type function struct {
115126
returnTypes []string
116127
}
117128

118-
type structAST struct {
119-
name string
120-
ast *ast.StructType
121-
}
122-
123129
func parseURLFromFunc(fn *ast.FuncDecl) string {
124130
for _, stmt := range fn.Body.List {
125131
if expr, ok := stmt.(*ast.AssignStmt); ok {
@@ -184,6 +190,57 @@ func getReturnTypes(fn *ast.FieldList) []string {
184190
return params
185191
}
186192

193+
func getStructValueNameFromResponse(pkgs map[string]*ast.Package, name string) (string, bool, bool) {
194+
st := getStruct(pkgs, name)
195+
st1name := st.Fields.List[0].Type.(*ast.Ident).Name
196+
st1 := getStruct(pkgs, st1name)
197+
structName := ""
198+
hasNextField := false
199+
for _, f := range st1.Fields.List {
200+
if f.Names[0].Name == "Value" {
201+
arrType := f.Type.(*ast.ArrayType)
202+
_, ok := arrType.Elt.(*ast.StarExpr)
203+
if !ok {
204+
structName = f.Type.(*ast.ArrayType).Elt.(*ast.Ident).Name
205+
} else {
206+
structName = f.Type.(*ast.ArrayType).Elt.(*ast.StarExpr).X.(*ast.Ident).Name
207+
}
208+
}
209+
if f.Names[0].Name == "NextLink" {
210+
hasNextField = true
211+
}
212+
}
213+
hasId := false
214+
st2 := getStruct(pkgs, structName)
215+
for _, f := range st2.Fields.List {
216+
if f.Names[0].Name == "ID" {
217+
hasId = true
218+
}
219+
}
220+
return structName, hasNextField, hasId
221+
}
222+
223+
func getStruct(pkgs map[string]*ast.Package, name string) *ast.StructType {
224+
for _, pack := range pkgs {
225+
for _, f := range pack.Files {
226+
for _, d := range f.Decls {
227+
if gen, isGen := d.(*ast.GenDecl); isGen {
228+
for _, spec := range gen.Specs {
229+
if typeSpec, isTypeSpec := spec.(*ast.TypeSpec); isTypeSpec {
230+
if structType, isStruct := typeSpec.Type.(*ast.StructType); isStruct {
231+
if typeSpec.Name.Name == name {
232+
return structType
233+
}
234+
}
235+
}
236+
}
237+
}
238+
}
239+
}
240+
}
241+
return nil
242+
}
243+
187244
// returns reciever and method name that matches re
188245
func findFunctions(pkgs map[string]*ast.Package, re *regexp.Regexp) map[string]*function {
189246
var funcs map[string]*function = make(map[string]*function)
@@ -254,7 +311,8 @@ func CreateTablesFromPackage(pkg string) ([]*Table, error) {
254311
}
255312
}
256313
tables[strings.TrimPrefix(fn.name, "New")] = &Table{
257-
NewFuncName: fn.name,
314+
NewFuncName: fn.name,
315+
NewClientParams: fn.paramNames,
258316
}
259317
}
260318
listMethods := findFunctions(pkgs, reListRequest)
@@ -312,8 +370,11 @@ func CreateTablesFromPackage(pkg string) ([]*Table, error) {
312370
} else {
313371
t.Multiplex = "client.SubscriptionMultiplex"
314372
}
315-
316-
result = append(result, t)
373+
hasId := false
374+
t.ResponseValueStruct, t.ResponspeStructNextLink, hasId = getStructValueNameFromResponse(pkgs, t.ResponseStruct)
375+
if hasId {
376+
result = append(result, t)
377+
}
317378
}
318379
sort.SliceStable(result, func(i, j int) bool {
319380
return result[i].NewFuncName < result[j].NewFuncName

‎plugins/source/azure/codegen0/internal/azparser/recipe.go‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,28 @@ package azparser
22

33
// Contains the information needed to generate CQ Table for specific Azure Client
44
type Table struct {
5+
// FullImportPath
6+
ImportPath string
7+
// Package in the plugin
8+
PackageName string
9+
BaseImportPath string
10+
// Name of the table
11+
Name string
512
// name of the function that creates a new azure client
613
NewFuncName string
714
// Namespace Parsed from the URL
815
Namespace string
16+
// NamespaceConst
17+
NamespaceConst string
918
// Pager name to use
1019
Pager string
1120
// ResponseStruct
12-
ResponseStruct string
21+
ResponseStruct string
22+
ResponspeStructNextLink bool
23+
ResponseValueStruct string
1324
// URL is the one set by NewListAll or NewList, depending on which one is available
14-
URL string
25+
URL string
26+
NewFuncHasSubscriptionId bool
1527
// param names for NewXClient function
1628
NewClientParams []string
1729
Multiplex string

0 commit comments

Comments
 (0)