Skip to content

Commit 3dd6116

Browse files
shimonp21shimonp21
authored
feat(gcp-resources): Add vision (#6786)
Caveats: - Generated only most-imporant/top-level tables (skipped productsets). - no programatic way to get list of supported locations. I copy-pasted them from the APIs error message: ``` 2:56PM ERR table resolver finished with error error="rpc error: code = InvalidArgument desc = Location not supported: 'europe-west3'. Must be one of 'us-west1', 'us-east1', 'asia-east1', 'europe-west1'" client=cq-playground module=gcp-src table=gcp_vision_product_sets ```
1 parent 7fad312 commit 3dd6116

14 files changed

Lines changed: 460 additions & 0 deletions

File tree

‎plugins/source/gcp/client/client.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ type Client struct {
4444
ProjectId string
4545
// this is set by table client Org multiplexer
4646
OrgId string
47+
// this is set by table client Location multiplexer
48+
Location string
4749
// Logger
4850
logger zerolog.Logger
4951
}
@@ -58,6 +60,13 @@ func (c *Client) withProject(project string) *Client {
5860
return &newClient
5961
}
6062

63+
func (c *Client) withLocation(location string) *Client {
64+
newClient := *c
65+
newClient.logger = c.logger.With().Str("location", location).Logger()
66+
newClient.Location = location
67+
return &newClient
68+
}
69+
6170
// withOrg allows multiplexer to create a new client with given organizationId
6271
func (c *Client) withOrg(org string) *Client {
6372
newClient := *c

‎plugins/source/gcp/client/multiplex.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,29 @@ func OrgMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
4444
}
4545
return l
4646
}
47+
48+
func ProjectLocationMultiplexEnabledServices(service string, locations []string) func(schema.ClientMeta) []schema.ClientMeta {
49+
if _, ok := GcpServices[service]; !ok {
50+
panic("unknown service: " + service)
51+
}
52+
53+
return func(meta schema.ClientMeta) []schema.ClientMeta {
54+
cl := meta.(*Client)
55+
56+
l := make([]schema.ClientMeta, 0, len(cl.projects))
57+
for _, projectId := range cl.projects {
58+
// This map can only be empty if user has not opted into `EnabledServicesOnly` via the spec
59+
if len(cl.EnabledServices) == 0 {
60+
for _, location := range locations {
61+
l = append(l, cl.withProject(projectId).withLocation(location))
62+
}
63+
} else if cl.EnabledServices[projectId] != nil && cl.EnabledServices[projectId][service] != nil {
64+
for _, location := range locations {
65+
l = append(l, cl.withProject(projectId).withLocation(location))
66+
}
67+
}
68+
}
69+
70+
return l
71+
}
72+
}

‎plugins/source/gcp/docs/tables/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@
143143
- [gcp_sql_users](gcp_sql_users.md)
144144
- [gcp_storage_buckets](gcp_storage_buckets.md)
145145
- [gcp_storage_bucket_policies](gcp_storage_bucket_policies.md)
146+
- [gcp_vision_products](gcp_vision_products.md)
147+
- [gcp_vision_product_reference_images](gcp_vision_product_reference_images.md)
146148
- [gcp_vmmigration_groups](gcp_vmmigration_groups.md)
147149
- [gcp_vmmigration_sources](gcp_vmmigration_sources.md)
148150
- [gcp_vmmigration_source_datacenter_connectors](gcp_vmmigration_source_datacenter_connectors.md)

‎plugins/source/gcp/docs/tables/gcp_vision_product_reference_images.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Table: gcp_vision_product_reference_images
2+
3+
https://cloud.google.com/vision/docs/reference/rest/v1/projects.locations.products.referenceImages
4+
5+
The composite primary key for this table is (**project_id**, **name**).
6+
7+
## Relations
8+
9+
This table depends on [gcp_vision_products](gcp_vision_products.md).
10+
11+
## Columns
12+
13+
| Name | Type |
14+
| ------------- | ------------- |
15+
|_cq_source_name|String|
16+
|_cq_sync_time|Timestamp|
17+
|_cq_id|UUID|
18+
|_cq_parent_id|UUID|
19+
|project_id (PK)|String|
20+
|name (PK)|String|
21+
|uri|String|
22+
|bounding_polys|JSON|

‎plugins/source/gcp/docs/tables/gcp_vision_products.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Table: gcp_vision_products
2+
3+
https://cloud.google.com/vision/docs/reference/rest/v1/projects.locations.products
4+
5+
The composite primary key for this table is (**project_id**, **name**).
6+
7+
## Relations
8+
9+
The following tables depend on gcp_vision_products:
10+
- [gcp_vision_product_reference_images](gcp_vision_product_reference_images.md)
11+
12+
## Columns
13+
14+
| Name | Type |
15+
| ------------- | ------------- |
16+
|_cq_source_name|String|
17+
|_cq_sync_time|Timestamp|
18+
|_cq_id|UUID|
19+
|_cq_parent_id|UUID|
20+
|project_id (PK)|String|
21+
|name (PK)|String|
22+
|display_name|String|
23+
|description|String|
24+
|product_category|String|
25+
|product_labels|JSON|

‎plugins/source/gcp/go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
cloud.google.com/go/secretmanager v1.9.0
3636
cloud.google.com/go/serviceusage v1.4.0
3737
cloud.google.com/go/storage v1.28.0
38+
cloud.google.com/go/vision/v2 v2.5.0
3839
cloud.google.com/go/vmmigration v1.3.0
3940
cloud.google.com/go/vpcaccess v1.5.0
4041
cloud.google.com/go/websecurityscanner v1.4.0
@@ -61,6 +62,7 @@ require (
6162
require (
6263
cloud.google.com/go v0.107.0 // indirect
6364
cloud.google.com/go/compute/metadata v0.2.2 // indirect
65+
cloud.google.com/go/vision v1.2.0 // indirect
6466
github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect
6567
github.com/cespare/xxhash/v2 v2.1.1 // indirect
6668
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect

‎plugins/source/gcp/go.sum‎

Lines changed: 171 additions & 0 deletions
Large diffs are not rendered by default.

‎plugins/source/gcp/resources/plugin/tables.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
"github.com/cloudquery/plugins/source/gcp/resources/services/serviceusage"
3737
"github.com/cloudquery/plugins/source/gcp/resources/services/sql"
3838
"github.com/cloudquery/plugins/source/gcp/resources/services/storage"
39+
"github.com/cloudquery/plugins/source/gcp/resources/services/vision"
3940
"github.com/cloudquery/plugins/source/gcp/resources/services/vmmigration"
4041
"github.com/cloudquery/plugins/source/gcp/resources/services/vpcaccess"
4142
"github.com/cloudquery/plugins/source/gcp/resources/services/websecurityscanner"
@@ -134,6 +135,7 @@ func PluginAutoGeneratedTables() []*schema.Table {
134135
serviceusage.Services(),
135136
sql.Instances(),
136137
storage.Buckets(),
138+
vision.Products(),
137139
vmmigration.Groups(),
138140
vmmigration.Sources(),
139141
vmmigration.TargetProjects(),

‎plugins/source/gcp/resources/services/vision/products.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package vision
2+
3+
import (
4+
pb "cloud.google.com/go/vision/v2/apiv1/visionpb"
5+
"github.com/cloudquery/plugin-sdk/schema"
6+
"github.com/cloudquery/plugin-sdk/transformers"
7+
"github.com/cloudquery/plugins/source/gcp/client"
8+
)
9+
10+
func Products() *schema.Table {
11+
return &schema.Table{
12+
Name: "gcp_vision_products",
13+
Description: `https://cloud.google.com/vision/docs/reference/rest/v1/projects.locations.products`,
14+
Resolver: fetchProducts,
15+
// The list of locations was copied by me from the APIs error message. Yuck.
16+
Multiplex: client.ProjectLocationMultiplexEnabledServices("vision.googleapis.com",
17+
[]string{
18+
"us-west1", "us-east1", "asia-east1", "europe-west1",
19+
}),
20+
Transform: transformers.TransformWithStruct(&pb.Product{}, client.Options()...),
21+
Columns: []schema.Column{
22+
{
23+
Name: "project_id",
24+
Type: schema.TypeString,
25+
Resolver: client.ResolveProject,
26+
CreationOptions: schema.ColumnCreationOptions{
27+
PrimaryKey: true,
28+
},
29+
},
30+
{
31+
Name: "name",
32+
Type: schema.TypeString,
33+
Resolver: schema.PathResolver("Name"),
34+
CreationOptions: schema.ColumnCreationOptions{
35+
PrimaryKey: true,
36+
},
37+
},
38+
},
39+
40+
Relations: []*schema.Table{
41+
ReferenceImages(),
42+
},
43+
}
44+
}

‎plugins/source/gcp/resources/services/vision/products_fetch.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package vision
2+
3+
import (
4+
"context"
5+
6+
vision "cloud.google.com/go/vision/v2/apiv1"
7+
pb "cloud.google.com/go/vision/v2/apiv1/visionpb"
8+
"github.com/cloudquery/plugin-sdk/schema"
9+
"github.com/cloudquery/plugins/source/gcp/client"
10+
"google.golang.org/api/iterator"
11+
)
12+
13+
func fetchProducts(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
14+
c := meta.(*client.Client)
15+
16+
gcpClient, err := vision.NewProductSearchClient(ctx, c.ClientOptions...)
17+
if err != nil {
18+
return err
19+
}
20+
21+
it := gcpClient.ListProducts(ctx, &pb.ListProductsRequest{
22+
Parent: "projects/" + c.ProjectId + "/locations/" + c.Location,
23+
PageSize: 100,
24+
}, c.CallOptions...)
25+
for {
26+
resp, err := it.Next()
27+
if err == iterator.Done {
28+
break
29+
}
30+
if err != nil {
31+
return err
32+
}
33+
34+
res <- resp
35+
}
36+
return nil
37+
}

0 commit comments

Comments
 (0)