Skip to content

Commit 1980d16

Browse files
erezrokaherezrokah
authored
feat(codegen): Add PrimaryKeys field to codegen (#5623)
 #### Summary Looks like we're using `ExtraColumns` mostly for setting primary keys, so I changed it so we only pass the names of primary keys. This allowed to me to remove some boilerplate around skipping the field, just so we can extra column it. Also this saves copy pasting the column type, resolver (which was missing in some cases), etc. **While the order of primary keys columns changed between all columns, the order between primary keys of the same table is the same**. I'm not sure how to classify this? `feat`? It is user facing, both in regards to columns and code gen. <!--
1 parent 9dfc991 commit 1980d16

File tree

77 files changed

+331
-509
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+331
-509
lines changed

‎plugins/source/gcp/codegen/main.go‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@ import (
2525
//go:embed templates/*.go.tpl
2626
var gcpTemplatesFS embed.FS
2727

28-
29-
3028
func main() {
3129

3230
for _, r := range recipes.Resources {
@@ -68,16 +66,7 @@ func generatePlugin(rr []*recipes.Resource) {
6866
}
6967

7068
func needsProjectIDColumn(r recipes.Resource) bool {
71-
if r.Multiplex == &recipes.OrgMultiplex {
72-
return false
73-
}
74-
75-
for _, c := range r.ExtraColumns {
76-
if c.Name == "project_id" {
77-
return false
78-
}
79-
}
80-
return true
69+
return r.Multiplex != &recipes.OrgMultiplex
8170
}
8271

8372
func generateResource(r recipes.Resource, mock bool) {
@@ -128,13 +117,9 @@ func generateResource(r recipes.Resource, mock bool) {
128117
r.MockImports = []string{reflect.TypeOf(r.Struct).Elem().PkgPath()}
129118
}
130119

131-
for _, f := range r.ExtraColumns {
132-
r.SkipFields = append(r.SkipFields, strcase.ToCamel(f.Name))
133-
}
134-
135120
extraColumns := r.ExtraColumns
136121
if needsProjectIDColumn(r) {
137-
extraColumns = append([]codegen.ColumnDefinition{recipes.ProjectIdColumn}, extraColumns...)
122+
extraColumns = append([]codegen.ColumnDefinition{recipes.ProjectIdColumn}, r.ExtraColumns...)
138123
}
139124

140125
opts := []codegen.TableOption{
@@ -187,6 +172,15 @@ func generateResource(r recipes.Resource, mock bool) {
187172
} else {
188173
r.Table.Multiplex = *r.Multiplex
189174
}
175+
176+
for _, f := range r.PrimaryKeys {
177+
for i := range r.Table.Columns {
178+
if r.Table.Columns[i].Name == f {
179+
r.Table.Columns[i].Options.PrimaryKey = true
180+
}
181+
}
182+
}
183+
190184
r.Table.Resolver = "fetch" + strcase.ToCamel(r.SubService)
191185
if r.PreResourceResolver != "" {
192186
r.Table.PreResourceResolver = r.PreResourceResolver

‎plugins/source/gcp/codegen/recipes/apikeys.go‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,15 @@ package recipes
22

33
import (
44
apikeys "cloud.google.com/go/apikeys/apiv2"
5-
"github.com/cloudquery/plugin-sdk/codegen"
6-
"github.com/cloudquery/plugin-sdk/schema"
75
pb "google.golang.org/genproto/googleapis/api/apikeys/v2"
86
)
97

10-
118
func init() {
129
resources := []*Resource{
1310
{
14-
SubService: "keys",
15-
Struct: &pb.Key{},
16-
SkipFields: []string{"Uid"},
17-
ExtraColumns: []codegen.ColumnDefinition{
18-
ProjectIdColumnPk,
19-
{
20-
Name: "uid",
21-
Type: schema.TypeString,
22-
Resolver: `schema.PathResolver("Uid")`,
23-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
24-
},
25-
},
26-
11+
SubService: "keys",
12+
Struct: &pb.Key{},
13+
PrimaryKeys: []string{ProjectIdColumn.Name, "uid"},
2714
ListFunction: (&apikeys.Client{}).ListKeys,
2815
RequestStruct: &pb.ListKeysRequest{},
2916
ResponseStruct: &pb.ListKeysResponse{},

‎plugins/source/gcp/codegen/recipes/base.go‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,8 @@ type Resource struct {
8080
// Don't generate fetch
8181
SkipFetch bool
8282
// SkipFields fields in go struct to skip when generating the table from the go struct
83-
SkipFields []string
84-
// ExtraColumns override, override generated columns
83+
SkipFields []string
84+
PrimaryKeys []string
8585
ExtraColumns []codegen.ColumnDefinition
8686
// NameTransformer custom name transformer for resource
8787
NameTransformer func(field reflect.StructField) (string, error)
@@ -93,13 +93,6 @@ var ProjectIdColumn = codegen.ColumnDefinition{
9393
Resolver: "client.ResolveProject",
9494
}
9595

96-
var ProjectIdColumnPk = codegen.ColumnDefinition{
97-
Name: "project_id",
98-
Type: schema.TypeString,
99-
Resolver: "client.ResolveProject",
100-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
101-
}
102-
10396
func CreateReplaceTransformer(replace map[string]string) func(field reflect.StructField) (string, error) {
10497
return func(field reflect.StructField) (string, error) {
10598
name, err := codegen.DefaultNameTransformer(field)

‎plugins/source/gcp/codegen/recipes/billing.go‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,8 @@ package recipes
33
import (
44
billing "cloud.google.com/go/billing/apiv1"
55
pb "cloud.google.com/go/billing/apiv1/billingpb"
6-
"github.com/cloudquery/plugin-sdk/codegen"
7-
"github.com/cloudquery/plugin-sdk/schema"
86
)
97

10-
11-
128
func init() {
139
resources := []*Resource{
1410
{
@@ -20,13 +16,7 @@ func init() {
2016
ListFunction: (&billing.CloudBillingClient{}).ListBillingAccounts,
2117
RegisterServer: pb.RegisterCloudBillingServer,
2218
UnimplementedServer: &pb.UnimplementedCloudBillingServer{},
23-
ExtraColumns: []codegen.ColumnDefinition{
24-
{
25-
Name: "name",
26-
Type: schema.TypeString,
27-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
28-
},
29-
},
19+
PrimaryKeys: []string{"name"},
3020
},
3121
{
3222
SubService: "services",
@@ -37,13 +27,7 @@ func init() {
3727
ListFunction: (&billing.CloudCatalogClient{}).ListServices,
3828
RegisterServer: pb.RegisterCloudCatalogServer,
3929
UnimplementedServer: &pb.UnimplementedCloudCatalogServer{},
40-
ExtraColumns: []codegen.ColumnDefinition{
41-
{
42-
Name: "name",
43-
Type: schema.TypeString,
44-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
45-
},
46-
},
30+
PrimaryKeys: []string{"name"},
4731
},
4832
}
4933

‎plugins/source/gcp/codegen/recipes/compute.go‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@ import (
66
"strings"
77

88
compute "cloud.google.com/go/compute/apiv1"
9-
"github.com/cloudquery/plugin-sdk/codegen"
10-
"github.com/cloudquery/plugin-sdk/schema"
119
"github.com/iancoleman/strcase"
1210
pb "google.golang.org/genproto/googleapis/cloud/compute/v1"
1311
)
@@ -227,14 +225,8 @@ func init() {
227225
}
228226
resource.MockImports = []string{"cloud.google.com/go/compute/apiv1"}
229227
resource.ProtobufImport = "google.golang.org/genproto/googleapis/cloud/compute/v1"
230-
if resource.ExtraColumns == nil {
231-
resource.ExtraColumns = []codegen.ColumnDefinition{
232-
{
233-
Name: "self_link",
234-
Type: schema.TypeString,
235-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
236-
},
237-
}
228+
if resource.PrimaryKeys == nil {
229+
resource.PrimaryKeys = []string{"self_link"}
238230
}
239231
}
240232

‎plugins/source/gcp/codegen/recipes/container.go‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,17 @@
11
package recipes
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/codegen"
5-
"github.com/cloudquery/plugin-sdk/schema"
64
pb "google.golang.org/genproto/googleapis/container/v1"
75
)
86

9-
10-
117
func init() {
128
resources := []*Resource{
139
{
14-
SubService: "clusters",
15-
Struct: &pb.Cluster{},
16-
SkipFetch: true,
17-
SkipMock: true,
18-
ExtraColumns: []codegen.ColumnDefinition{
19-
{
20-
Name: "self_link",
21-
Type: schema.TypeString,
22-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
23-
},
24-
},
10+
SubService: "clusters",
11+
Struct: &pb.Cluster{},
12+
SkipFetch: true,
13+
SkipMock: true,
14+
PrimaryKeys: []string{"self_link"},
2515
NameTransformer: CreateReplaceTransformer(map[string]string{"ipv_4": "ipv4"}),
2616
},
2717
}

‎plugins/source/gcp/codegen/recipes/containeranalysis.go‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,14 @@ package recipes
33
import (
44
containeranalysis "cloud.google.com/go/containeranalysis/apiv1beta1"
55
grafeaspb "cloud.google.com/go/containeranalysis/apiv1beta1/grafeas/grafeaspb"
6-
"github.com/cloudquery/plugin-sdk/codegen"
7-
"github.com/cloudquery/plugin-sdk/schema"
86
)
97

10-
func init(){
8+
func init() {
119
resources := []*Resource{
1210
{
13-
SubService: "occurrences",
14-
Struct: &grafeaspb.Occurrence{},
15-
SkipFields: []string{"Name"},
16-
ExtraColumns: []codegen.ColumnDefinition{
17-
ProjectIdColumn,
18-
{
19-
Name: "name",
20-
Type: schema.TypeString,
21-
Resolver: `schema.PathResolver("Name")`,
22-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
23-
},
24-
},
11+
SubService: "occurrences",
12+
Struct: &grafeaspb.Occurrence{},
13+
PrimaryKeys: []string{"name"},
2514
Template: "newapi_list",
2615
ListFunction: (&containeranalysis.GrafeasV1Beta1Client{}).ListOccurrences,
2716
RequestStruct: &grafeaspb.ListOccurrencesRequest{},

‎plugins/source/gcp/codegen/recipes/dns.go‎

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,25 @@
11
package recipes
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/codegen"
5-
"github.com/cloudquery/plugin-sdk/schema"
64
"github.com/iancoleman/strcase"
75
"google.golang.org/api/dns/v1"
86
)
97

10-
11-
128
func init() {
139
resources := []*Resource{
1410
{
1511
SubService: "policies",
1612
Struct: &dns.Policy{},
1713
NewFunction: dns.NewService,
1814
ListFunction: (&dns.PoliciesService{}).List,
19-
ExtraColumns: []codegen.ColumnDefinition{
20-
{
21-
Name: "id",
22-
Type: schema.TypeInt,
23-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
24-
Resolver: `schema.PathResolver("Id")`,
25-
},
26-
},
15+
PrimaryKeys: []string{"id"},
2716
},
2817
{
2918
SubService: "managed_zones",
3019
Struct: &dns.ManagedZone{},
3120
NewFunction: dns.NewManagedZoneOperationsService,
3221
ListFunction: (&dns.ManagedZonesService{}).List,
33-
ExtraColumns: []codegen.ColumnDefinition{
34-
{
35-
Name: "id",
36-
Type: schema.TypeInt,
37-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
38-
Resolver: `schema.PathResolver("Id")`,
39-
},
40-
},
22+
PrimaryKeys: []string{"id"},
4123
},
4224
}
4325

‎plugins/source/gcp/codegen/recipes/iam.go‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,39 +7,18 @@ import (
77
"google.golang.org/api/iam/v1"
88
)
99

10-
11-
1210
func init() {
1311
resources := []*Resource{
1412
{
15-
SubService: "roles",
16-
Struct: &iam.Role{},
17-
ExtraColumns: []codegen.ColumnDefinition{
18-
{
19-
Name: "project_id",
20-
Type: schema.TypeString,
21-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
22-
Resolver: "client.ResolveProject",
23-
},
24-
{
25-
Name: "name",
26-
Type: schema.TypeString,
27-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
28-
},
29-
},
13+
SubService: "roles",
14+
Struct: &iam.Role{},
15+
PrimaryKeys: []string{ProjectIdColumn.Name, "name"},
3016
},
3117
{
32-
SubService: "service_accounts",
33-
Struct: &iam.ServiceAccount{},
34-
OutputField: "Accounts",
35-
ExtraColumns: []codegen.ColumnDefinition{
36-
{
37-
Name: "unique_id",
38-
Type: schema.TypeString,
39-
Options: schema.ColumnCreationOptions{PrimaryKey: true},
40-
Resolver: `schema.PathResolver("UniqueId")`,
41-
},
42-
},
18+
SubService: "service_accounts",
19+
Struct: &iam.ServiceAccount{},
20+
OutputField: "Accounts",
21+
PrimaryKeys: []string{"unique_id"},
4322
SkipFields: []string{"ProjectId"},
4423
NameTransformer: CreateReplaceTransformer(map[string]string{"oauth_2": "oauth2"}),
4524
Relations: []string{"ServiceAccountKeys()"},

‎plugins/source/gcp/codegen/recipes/kms.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ func init() {
1717
ChildTable: true,
1818
SkipMock: true,
1919
SkipFetch: true,
20-
SkipFields: []string{"RotationSchedule"},
2120
ExtraColumns: codegen.ColumnDefinitions{
2221
{
2322
Name: "rotation_period",

0 commit comments

Comments
 (0)