Skip to content

Commit 1a964ac

Browse files
authored
feat: Add codegen (#5618)
#### Summary Follow up to #5611, related to #5511. This adds basic code gen for AzureDevOps (only tables). It also has `projects` generated, still with no primary key (not sure if `Id` is unique across the API, we'll need to check) <!--
1 parent d02d4e7 commit 1a964ac

12 files changed

Lines changed: 297 additions & 34 deletions

File tree

plugins/source/azuredevops/Makefile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,10 @@ gen-docs:
1212
rm -rf ./docs/tables/*
1313
go run main.go doc ./docs/tables
1414

15+
.PHONY: gen-code
16+
gen-code:
17+
grep -rl '// Code generated by codegen; DO NOT EDIT.' resources/services/* | xargs rm
18+
go run codegen/main.go
19+
1520
.PHONY: gen
16-
gen: gen-docs
21+
gen: gen-code gen-docs
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package main
2+
3+
import (
4+
"log"
5+
6+
"github.com/cloudquery/cloudquery/plugins/source/azuredevops/codegen/recipes"
7+
)
8+
9+
func main() {
10+
for _, resource := range recipes.Resources {
11+
if err := resource.Generate(); err != nil {
12+
log.Fatal(err)
13+
}
14+
}
15+
if err := recipes.GenerateTablesList(recipes.Resources); err != nil {
16+
log.Fatal(err)
17+
}
18+
}
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package recipes
2+
3+
import (
4+
"bytes"
5+
"embed"
6+
"fmt"
7+
"go/format"
8+
"os"
9+
"path"
10+
"reflect"
11+
"runtime"
12+
"text/template"
13+
14+
"github.com/cloudquery/plugin-sdk/codegen"
15+
"github.com/cloudquery/plugin-sdk/schema"
16+
"github.com/google/uuid"
17+
"github.com/iancoleman/strcase"
18+
)
19+
20+
//go:embed templates/*.go.tpl
21+
var templatesFS embed.FS
22+
var Resources []*Resource
23+
24+
type Resource struct {
25+
Service string
26+
SubService string
27+
Struct interface{}
28+
Table *codegen.TableDefinition
29+
}
30+
31+
func writeTemplateContentToFile(dir string, filePath string, buff bytes.Buffer) error {
32+
outputPath := path.Join(dir, "../..", filePath)
33+
outputDir := path.Dir(outputPath)
34+
if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
35+
return fmt.Errorf("failed to create directory %s: %w", outputDir, err)
36+
}
37+
38+
content := buff.Bytes()
39+
formattedContent, err := format.Source(content)
40+
if err != nil {
41+
fmt.Printf("failed to format source: %s: %v\n", filePath, err)
42+
} else {
43+
content = formattedContent
44+
}
45+
46+
if err := os.WriteFile(outputPath, content, 0644); err != nil {
47+
return fmt.Errorf("failed to write file %s: %w", filePath, err)
48+
}
49+
return nil
50+
}
51+
52+
func renderTemplate(name string, filePath string, data interface{}) error {
53+
_, filename, _, ok := runtime.Caller(0)
54+
if !ok {
55+
return fmt.Errorf("failed to get caller information")
56+
}
57+
dir := path.Dir(filename)
58+
59+
tpl, err := template.New(name).Funcs(template.FuncMap{"ToCamel": strcase.ToCamel}).ParseFS(templatesFS, "templates/*.go.tpl")
60+
if err != nil {
61+
return fmt.Errorf("failed to parse azureDevops templates: %w", err)
62+
}
63+
tpl, err = tpl.ParseFS(codegen.TemplatesFS, "templates/*.go.tpl")
64+
if err != nil {
65+
return fmt.Errorf("failed to parse sdk template: %w", err)
66+
}
67+
68+
var buff bytes.Buffer
69+
if err := tpl.Execute(&buff, data); err != nil {
70+
return fmt.Errorf("failed to execute template: %w", err)
71+
}
72+
73+
return writeTemplateContentToFile(dir, filePath, buff)
74+
}
75+
76+
func (resource *Resource) generate() error {
77+
return renderTemplate("resource.go.tpl", path.Join("resources", "services", resource.Service, resource.SubService+".go"), resource)
78+
}
79+
80+
func isUUID(fieldType reflect.Type) bool {
81+
fieldKind := fieldType.Kind()
82+
83+
if fieldKind == reflect.Ptr {
84+
return isUUID(fieldType.Elem())
85+
}
86+
87+
return fieldType == reflect.TypeOf(uuid.UUID{})
88+
}
89+
90+
func typeTransformer(field reflect.StructField) (schema.ValueType, error) {
91+
if isUUID(field.Type) {
92+
return schema.TypeUUID, nil
93+
}
94+
95+
return codegen.DefaultTypeTransformer(field)
96+
}
97+
98+
func (resource *Resource) Generate() error {
99+
var err error
100+
resource.Table, err = codegen.NewTableFromStruct(
101+
fmt.Sprintf("azuredevops_%s_%s", resource.Service, resource.SubService),
102+
resource.Struct,
103+
codegen.WithTypeTransformer(typeTransformer),
104+
)
105+
106+
if err != nil {
107+
return err
108+
}
109+
110+
resource.Table.Resolver = "fetch" + strcase.ToCamel(resource.SubService)
111+
if err := resource.generate(); err != nil {
112+
return err
113+
}
114+
115+
return nil
116+
}
117+
118+
func GenerateTablesList(resources []*Resource) error {
119+
return renderTemplate("tables.go.tpl", path.Join("resources", "plugin", "tables.go"), resources)
120+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package recipes
2+
3+
import (
4+
"github.com/microsoft/azure-devops-go-api/azuredevops/v6/core"
5+
)
6+
7+
func init() {
8+
resources := []*Resource{
9+
{
10+
SubService: "projects",
11+
Struct: &core.TeamProjectReference{},
12+
},
13+
}
14+
15+
for _, resource := range resources {
16+
resource.Service = "core"
17+
}
18+
19+
Resources = append(Resources, resources...)
20+
}
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Code generated by codegen; DO NOT EDIT.
2+
3+
package {{.Service}}
4+
5+
import (
6+
"github.com/cloudquery/plugin-sdk/schema"
7+
)
8+
9+
func {{.SubService | ToCamel}}() *schema.Table {
10+
return &schema.Table{{template "table.go.tpl" .Table}}
11+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Code generated by codegen; DO NOT EDIT.
2+
3+
package plugin
4+
5+
import (
6+
{{- range .}}
7+
"github.com/cloudquery/cloudquery/plugins/source/azuredevops/resources/services/{{.Service}}"
8+
{{- end}}
9+
"github.com/cloudquery/plugin-sdk/schema"
10+
)
11+
12+
func tables() []*schema.Table {
13+
return []*schema.Table{
14+
{{- range .}}
15+
{{.Service}}.{{.SubService | ToCamel}}(),
16+
{{- end}}
17+
}
18+
}

plugins/source/azuredevops/docs/tables/azuredevops_core_projects.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,13 @@ The primary key for this table is **_cq_id**.
1313
|_cq_sync_time|Timestamp|
1414
|_cq_id (PK)|UUID|
1515
|_cq_parent_id|UUID|
16-
|id|UUID|
16+
|abbreviation|String|
17+
|default_team_image_url|String|
18+
|description|String|
19+
|id|UUID|
20+
|last_update_time|JSON|
21+
|name|String|
22+
|revision|Int|
23+
|state|String|
24+
|url|String|
25+
|visibility|String|

plugins/source/azuredevops/go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ go 1.19
44

55
require (
66
github.com/cloudquery/plugin-sdk v1.12.5
7+
github.com/google/uuid v1.3.0
8+
github.com/iancoleman/strcase v0.2.0
79
github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1
810
github.com/rs/zerolog v1.28.0
911
)
@@ -12,7 +14,6 @@ require (
1214
github.com/getsentry/sentry-go v0.15.0 // indirect
1315
github.com/ghodss/yaml v1.0.0 // indirect
1416
github.com/golang/protobuf v1.5.2 // indirect
15-
github.com/google/uuid v1.3.0 // indirect
1617
github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.3 // indirect
1718
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
1819
github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -22,6 +23,7 @@ require (
2223
github.com/spf13/cobra v1.6.1 // indirect
2324
github.com/spf13/pflag v1.0.5 // indirect
2425
github.com/thoas/go-funk v0.9.2 // indirect
26+
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 // indirect
2527
golang.org/x/net v0.2.0 // indirect
2628
golang.org/x/sync v0.1.0 // indirect
2729
golang.org/x/sys v0.2.0 // indirect

plugins/source/azuredevops/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5
126126
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E=
127127
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
128128
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
129+
github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
130+
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
129131
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
130132
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
131133
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -202,6 +204,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
202204
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
203205
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
204206
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
207+
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4=
208+
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
205209
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
206210
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
207211
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

plugins/source/azuredevops/resources/plugin/tables.go

Lines changed: 5 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)