Skip to content

Commit 8fa403f

Browse files
DougEverlyDougEverly
authored
feat(resources): Add support for GCP organization and project tags (#14638)
 #### Sync GCP Organization and Project Tags * Sync tagKeys and tagValues for organizations * Sync tagKeys, tagValues, and tagBindings for projects. <!--
1 parent 715bfe8 commit 8fa403f

15 files changed

+565
-0
lines changed

‎plugins/source/gcp/client/services.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/gcp/docs/tables/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,12 @@
146146
- [gcp_redis_instances](../../../../../website/tables/gcp/gcp_redis_instances.md)
147147
- [gcp_resourcemanager_folders](../../../../../website/tables/gcp/gcp_resourcemanager_folders.md)
148148
- [gcp_resourcemanager_subfolders](../../../../../website/tables/gcp/gcp_resourcemanager_subfolders.md)
149+
- [gcp_resourcemanager_organization_tag_keys](../../../../../website/tables/gcp/gcp_resourcemanager_organization_tag_keys.md)
150+
- [gcp_resourcemanager_organization_tag_values](../../../../../website/tables/gcp/gcp_resourcemanager_organization_tag_values.md)
149151
- [gcp_resourcemanager_project_policies](../../../../../website/tables/gcp/gcp_resourcemanager_project_policies.md)
152+
- [gcp_resourcemanager_project_tag_bindings](../../../../../website/tables/gcp/gcp_resourcemanager_project_tag_bindings.md)
153+
- [gcp_resourcemanager_project_tag_keys](../../../../../website/tables/gcp/gcp_resourcemanager_project_tag_keys.md)
154+
- [gcp_resourcemanager_project_tag_values](../../../../../website/tables/gcp/gcp_resourcemanager_project_tag_values.md)
150155
- [gcp_resourcemanager_projects](../../../../../website/tables/gcp/gcp_resourcemanager_projects.md)
151156
- [gcp_run_locations](../../../../../website/tables/gcp/gcp_run_locations.md)
152157
- [gcp_run_services](../../../../../website/tables/gcp/gcp_run_services.md)

‎plugins/source/gcp/resources/plugin/tables.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ func PluginAutoGeneratedTables() schema.Tables {
145145
resourcemanager.Folders(),
146146
resourcemanager.Projects(),
147147
resourcemanager.ProjectPolicies(),
148+
resourcemanager.OrganizationTagKeys(),
149+
resourcemanager.ProjectTagKeys(),
150+
resourcemanager.ProjectTagBindings(),
148151
run.Locations(),
149152
secretmanager.Secrets(),
150153
securitycenter.ProjectFindings(),

‎plugins/source/gcp/resources/services/resourcemanager/organization_tags.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package resourcemanager
2+
3+
import (
4+
pb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
5+
"github.com/apache/arrow/go/v14/arrow"
6+
"github.com/cloudquery/cloudquery/plugins/source/gcp/client"
7+
"github.com/cloudquery/plugin-sdk/v4/schema"
8+
"github.com/cloudquery/plugin-sdk/v4/transformers"
9+
)
10+
11+
func OrganizationTagKeys() *schema.Table {
12+
return &schema.Table{
13+
Name: "gcp_resourcemanager_organization_tag_keys",
14+
Description: `https://cloud.google.com/resource-manager/reference/rest/v3/tagKeys/list`,
15+
Resolver: fetchOrganizationTagKeys,
16+
Multiplex: client.OrgMultiplex,
17+
Transform: client.TransformWithStruct(&pb.TagKey{}, transformers.WithPrimaryKeys("Name")),
18+
Columns: []schema.Column{
19+
{
20+
Name: "organization_id",
21+
Type: arrow.BinaryTypes.String,
22+
Resolver: client.ResolveOrganization,
23+
PrimaryKey: true,
24+
},
25+
},
26+
Relations: []*schema.Table{
27+
organizationTagValues(),
28+
},
29+
}
30+
}
31+
32+
func organizationTagValues() *schema.Table {
33+
return &schema.Table{
34+
Name: "gcp_resourcemanager_organization_tag_values",
35+
Description: `https://cloud.google.com/resource-manager/reference/rest/v3/tagValues/list`,
36+
Resolver: fetchOrganizationTagValues,
37+
Multiplex: client.OrgMultiplex,
38+
Transform: client.TransformWithStruct(&pb.TagValue{}, transformers.WithPrimaryKeys("Name")),
39+
Columns: []schema.Column{
40+
{
41+
Name: "organization_id",
42+
Type: arrow.BinaryTypes.String,
43+
Resolver: client.ResolveOrganization,
44+
PrimaryKey: true,
45+
},
46+
},
47+
}
48+
}

‎plugins/source/gcp/resources/services/resourcemanager/organization_tags_fetch.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package resourcemanager
2+
3+
import (
4+
"context"
5+
6+
resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
7+
pb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
8+
"github.com/cloudquery/cloudquery/plugins/source/gcp/client"
9+
"github.com/cloudquery/plugin-sdk/v4/schema"
10+
"google.golang.org/api/iterator"
11+
)
12+
13+
func fetchOrganizationTagKeys(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
14+
c := meta.(*client.Client)
15+
16+
fClient, err := resourcemanager.NewTagKeysClient(ctx, c.ClientOptions...)
17+
if err != nil {
18+
return err
19+
}
20+
21+
req := &pb.ListTagKeysRequest{
22+
Parent: "organizations/" + c.OrgId,
23+
}
24+
it := fClient.ListTagKeys(ctx, req)
25+
for {
26+
resp, err := it.Next()
27+
if err == iterator.Done {
28+
break
29+
}
30+
if err != nil {
31+
return err
32+
}
33+
res <- resp
34+
}
35+
return nil
36+
}
37+
38+
func fetchOrganizationTagValues(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
39+
c := meta.(*client.Client)
40+
41+
fClient, err := resourcemanager.NewTagValuesClient(ctx, c.ClientOptions...)
42+
if err != nil {
43+
return err
44+
}
45+
46+
var parentName string
47+
if parent != nil {
48+
parentName = parent.Item.(*pb.TagKey).Name
49+
}
50+
req := &pb.ListTagValuesRequest{
51+
Parent: parentName,
52+
}
53+
54+
it := fClient.ListTagValues(ctx, req)
55+
for {
56+
resp, err := it.Next()
57+
if err == iterator.Done {
58+
break
59+
}
60+
if err != nil {
61+
return err
62+
}
63+
64+
res <- resp
65+
}
66+
return nil
67+
}

‎plugins/source/gcp/resources/services/resourcemanager/organization_tags_mock_test.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package resourcemanager
2+
3+
import (
4+
pb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
5+
"context"
6+
"fmt"
7+
"github.com/cloudquery/cloudquery/plugins/source/gcp/client"
8+
"github.com/cloudquery/plugin-sdk/v4/faker"
9+
"google.golang.org/grpc"
10+
"testing"
11+
)
12+
13+
func createOrganizationTagKeys(gsrv *grpc.Server) error {
14+
fakeServer := &fakeOrganizationTagKeysServer{}
15+
pb.RegisterTagKeysServer(gsrv, fakeServer)
16+
return createOrganizationTagValues(gsrv)
17+
}
18+
19+
type fakeOrganizationTagKeysServer struct {
20+
pb.UnimplementedTagKeysServer
21+
}
22+
23+
func (*fakeOrganizationTagKeysServer) ListTagKeys(_ context.Context, req *pb.ListTagKeysRequest) (*pb.ListTagKeysResponse, error) {
24+
resp := pb.ListTagKeysResponse{}
25+
if err := faker.FakeObject(&resp); err != nil {
26+
return nil, fmt.Errorf("failed to fake data: %w", err)
27+
}
28+
29+
for _, f := range resp.TagKeys {
30+
f.Name = "tagKeys/456"
31+
f.Parent = req.Parent
32+
}
33+
34+
resp.NextPageToken = ""
35+
return &resp, nil
36+
}
37+
38+
func TestOrganizationTagKeys(t *testing.T) {
39+
client.MockTestGrpcHelper(t, OrganizationTagKeys(), createOrganizationTagKeys, client.TestOptions{})
40+
}
41+
42+
func createOrganizationTagValues(gsrv *grpc.Server) error {
43+
fakeServer := &fakeOrganizationTagValuesServer{}
44+
pb.RegisterTagValuesServer(gsrv, fakeServer)
45+
return nil
46+
}
47+
48+
type fakeOrganizationTagValuesServer struct {
49+
pb.UnimplementedTagValuesServer
50+
}
51+
52+
func (*fakeOrganizationTagValuesServer) ListTagValues(_ context.Context, req *pb.ListTagValuesRequest) (*pb.ListTagValuesResponse, error) {
53+
resp := pb.ListTagValuesResponse{}
54+
if err := faker.FakeObject(&resp); err != nil {
55+
return nil, fmt.Errorf("failed to fake data: %w", err)
56+
}
57+
58+
for _, f := range resp.TagValues {
59+
f.Name = "tagValues/789"
60+
f.Parent = req.Parent
61+
}
62+
63+
resp.NextPageToken = ""
64+
return &resp, nil
65+
}
66+
67+
func TestOrganizationTagValues(t *testing.T) {
68+
client.MockTestGrpcHelper(t, organizationTagValues(), createOrganizationTagValues, client.TestOptions{})
69+
}

‎plugins/source/gcp/resources/services/resourcemanager/project_tags.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package resourcemanager
2+
3+
import (
4+
pb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
5+
"github.com/apache/arrow/go/v14/arrow"
6+
"github.com/cloudquery/cloudquery/plugins/source/gcp/client"
7+
"github.com/cloudquery/plugin-sdk/v4/schema"
8+
"github.com/cloudquery/plugin-sdk/v4/transformers"
9+
)
10+
11+
func ProjectTagKeys() *schema.Table {
12+
return &schema.Table{
13+
Name: "gcp_resourcemanager_project_tag_keys",
14+
Description: `https://cloud.google.com/resource-manager/reference/rest/v3/tagKeys/list`,
15+
Resolver: fetchProjectTagKeys,
16+
Multiplex: client.ProjectMultiplexEnabledServices("cloudresourcemanager.googleapis.com"),
17+
Transform: client.TransformWithStruct(&pb.TagKey{}, transformers.WithPrimaryKeys("Name")),
18+
Columns: []schema.Column{
19+
{
20+
Name: "project_id",
21+
Type: arrow.BinaryTypes.String,
22+
Resolver: client.ResolveProject,
23+
PrimaryKey: true,
24+
},
25+
},
26+
Relations: []*schema.Table{
27+
projectTagValues(),
28+
},
29+
}
30+
}
31+
32+
func projectTagValues() *schema.Table {
33+
return &schema.Table{
34+
Name: "gcp_resourcemanager_project_tag_values",
35+
Description: `https://cloud.google.com/resource-manager/reference/rest/v3/tagValues/list`,
36+
Resolver: fetchProjectTagValues,
37+
Multiplex: client.ProjectMultiplexEnabledServices("resourcemanager.googleapis.com"),
38+
Transform: client.TransformWithStruct(&pb.TagValue{}, transformers.WithPrimaryKeys("Name")),
39+
Columns: []schema.Column{
40+
{
41+
Name: "project_id",
42+
Type: arrow.BinaryTypes.String,
43+
Resolver: client.ResolveProject,
44+
PrimaryKey: true,
45+
},
46+
},
47+
}
48+
}
49+
50+
func ProjectTagBindings() *schema.Table {
51+
return &schema.Table{
52+
Name: "gcp_resourcemanager_project_tag_bindings",
53+
Description: `https://cloud.google.com/resource-manager/reference/rest/v3/tagBindings`,
54+
Resolver: fetchProjectTagBindings,
55+
Multiplex: client.ProjectMultiplexEnabledServices("cloudresourcemanager.googleapis.com"),
56+
Transform: client.TransformWithStruct(&pb.TagValue{}, transformers.WithPrimaryKeys("Name")),
57+
Columns: []schema.Column{
58+
{
59+
Name: "project_id",
60+
Type: arrow.BinaryTypes.String,
61+
Resolver: client.ResolveProject,
62+
PrimaryKey: true,
63+
},
64+
},
65+
}
66+
}

‎plugins/source/gcp/resources/services/resourcemanager/project_tags_fetch.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package resourcemanager
2+
3+
import (
4+
resourcemanager "cloud.google.com/go/resourcemanager/apiv3"
5+
pb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
6+
"context"
7+
"github.com/cloudquery/cloudquery/plugins/source/gcp/client"
8+
"github.com/cloudquery/plugin-sdk/v4/schema"
9+
"google.golang.org/api/iterator"
10+
)
11+
12+
func fetchProjectTagKeys(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error {
13+
c := meta.(*client.Client)
14+
15+
fClient, err := resourcemanager.NewTagKeysClient(ctx, c.ClientOptions...)
16+
if err != nil {
17+
return err
18+
}
19+
20+
req := &pb.ListTagKeysRequest{
21+
Parent: "projects/" + c.ProjectId,
22+
}
23+
it := fClient.ListTagKeys(ctx, req)
24+
for {
25+
resp, err := it.Next()
26+
if err == iterator.Done {
27+
break
28+
}
29+
if err != nil {
30+
return err
31+
}
32+
33+
res <- resp
34+
}
35+
return nil
36+
}
37+
38+
func fetchProjectTagValues(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
39+
c := meta.(*client.Client)
40+
41+
fClient, err := resourcemanager.NewTagValuesClient(ctx, c.ClientOptions...)
42+
if err != nil {
43+
return err
44+
}
45+
46+
var parentName string
47+
if parent != nil {
48+
parentName = parent.Item.(*pb.TagKey).Name
49+
}
50+
req := &pb.ListTagValuesRequest{
51+
Parent: parentName,
52+
}
53+
54+
it := fClient.ListTagValues(ctx, req)
55+
for {
56+
resp, err := it.Next()
57+
if err == iterator.Done {
58+
break
59+
}
60+
if err != nil {
61+
return err
62+
}
63+
64+
res <- resp
65+
}
66+
return nil
67+
}
68+
69+
func fetchProjectTagBindings(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- any) error {
70+
c := meta.(*client.Client)
71+
72+
fClient, err := resourcemanager.NewTagBindingsClient(ctx, c.ClientOptions...)
73+
if err != nil {
74+
return err
75+
}
76+
77+
req := &pb.ListTagBindingsRequest{
78+
Parent: "//cloudresourcemanager.googleapis.com/projects/" + c.ProjectId,
79+
}
80+
81+
it := fClient.ListTagBindings(ctx, req)
82+
for {
83+
resp, err := it.Next()
84+
if err == iterator.Done {
85+
break
86+
}
87+
if err != nil {
88+
return err
89+
}
90+
91+
res <- resp
92+
}
93+
return nil
94+
}

0 commit comments

Comments
 (0)