Skip to content

Commit 340f614

Browse files
disqdisq
authored
feat(aws): Use PreResourceResolver when necessary, remove ListAndDetailResolver (#2460)
More here #2461 **BEWARE** merging this before updating the plugin-sdk for "PreResourceResolver parallelism" will cause performance regressions. (#2461 doesn't have that issue, as those resources are already linear)
1 parent 107a0f4 commit 340f614

32 files changed

+356
-482
lines changed

‎plugins/source/aws/client/helpers.go‎

Lines changed: 0 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"github.com/aws/aws-sdk-go-v2/aws/arn"
1616
"github.com/aws/smithy-go"
1717
"github.com/cloudquery/plugin-sdk/schema"
18-
"golang.org/x/sync/semaphore"
1918
)
2019

2120
type AWSService string
@@ -63,8 +62,6 @@ const (
6362
WorkspacesService AWSService = "workspaces"
6463
)
6564

66-
const MAX_GOROUTINES = 10
67-
6865
const (
6966
PartitionServiceRegionFile = "data/partition_service_region.json"
7067
defaultPartition = "aws"
@@ -419,45 +416,6 @@ func TagsToMap(tagSlice interface{}) map[string]string {
419416
return ret
420417
}
421418

422-
func ListAndDetailResolver(ctx context.Context, meta schema.ClientMeta, res chan<- interface{}, list ListResolverFunc, details DetailResolverFunc) error {
423-
errorChan := make(chan error)
424-
detailChan := make(chan interface{})
425-
c := meta.(*Client)
426-
// Channel that will communicate with goroutine that is aggregating the errors
427-
done := make(chan struct{})
428-
go func() {
429-
defer close(done)
430-
for detailError := range errorChan {
431-
c.logger.Warn().Err(detailError).Msg("Error while fetching details")
432-
}
433-
}()
434-
sem := semaphore.NewWeighted(int64(MAX_GOROUTINES))
435-
436-
go func() {
437-
defer close(errorChan)
438-
for item := range detailChan {
439-
if err := sem.Acquire(ctx, 1); err != nil {
440-
continue
441-
}
442-
func(summary interface{}) {
443-
defer sem.Release(1)
444-
details(ctx, meta, res, errorChan, summary)
445-
}(item)
446-
}
447-
}()
448-
449-
err := list(ctx, meta, detailChan)
450-
close(detailChan)
451-
if err != nil {
452-
return err
453-
}
454-
455-
// All items will be attempted to be fetched, and all errors will be aggregated
456-
<-done
457-
458-
return nil
459-
}
460-
461419
func Sleep(ctx context.Context, dur time.Duration) error {
462420
select {
463421
case <-ctx.Done():

‎plugins/source/aws/codegen/recipes/athena.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import (
99
func AthenaResources() []*Resource {
1010
resources := []*Resource{
1111
{
12-
SubService: "data_catalogs",
13-
Struct: &types.DataCatalog{},
14-
SkipFields: []string{},
12+
SubService: "data_catalogs",
13+
Struct: &types.DataCatalog{},
14+
SkipFields: []string{},
15+
PreResourceResolver: "getDataCatalog",
1516
ExtraColumns: append(
1617
defaultRegionalColumns,
1718
[]codegen.ColumnDefinition{
@@ -83,9 +84,10 @@ func AthenaResources() []*Resource {
8384
}...),
8485
},
8586
{
86-
SubService: "work_groups",
87-
Struct: &types.WorkGroup{},
88-
SkipFields: []string{},
87+
SubService: "work_groups",
88+
Struct: &types.WorkGroup{},
89+
SkipFields: []string{},
90+
PreResourceResolver: "getWorkGroup",
8991
ExtraColumns: append(
9092
defaultRegionalColumns,
9193
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/ecs.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,10 @@ func ECSResources() []*Resource {
100100
Relations: []string{},
101101
},
102102
{
103-
SubService: "task_definitions",
104-
Struct: &models.TaskDefinitionWrapper{},
105-
SkipFields: []string{"TaskDefinitionArn", "Tags"},
103+
SubService: "task_definitions",
104+
Struct: &models.TaskDefinitionWrapper{},
105+
SkipFields: []string{"TaskDefinitionArn", "Tags"},
106+
PreResourceResolver: "getTaskDefinition",
106107
ExtraColumns: append(
107108
defaultRegionalColumns,
108109
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/firehose.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import (
99
func FirehoseResources() []*Resource {
1010
resources := []*Resource{
1111
{
12-
SubService: "delivery_streams",
13-
Struct: &types.DeliveryStreamDescription{},
14-
SkipFields: []string{"DeliveryStreamARN"},
12+
SubService: "delivery_streams",
13+
Struct: &types.DeliveryStreamDescription{},
14+
SkipFields: []string{"DeliveryStreamARN"},
15+
PreResourceResolver: "getDeliveryStream",
1516
ExtraColumns: append(
1617
defaultRegionalColumns,
1718
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/glue.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -351,9 +351,10 @@ func GlueResources() []*Resource {
351351
},
352352
},
353353
{
354-
SubService: "triggers",
355-
Struct: &types.Trigger{},
356-
SkipFields: []string{},
354+
SubService: "triggers",
355+
Struct: &types.Trigger{},
356+
SkipFields: []string{},
357+
PreResourceResolver: "getTrigger",
357358
ExtraColumns: append(
358359
defaultRegionalColumns,
359360
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/iam.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -216,9 +216,10 @@ func IAMResources() []*Resource {
216216
},
217217
},
218218
{
219-
SubService: "roles",
220-
Struct: &types.Role{},
221-
SkipFields: []string{"RoleId", "Tags", "AssumeRolePolicyDocument"},
219+
SubService: "roles",
220+
Struct: &types.Role{},
221+
SkipFields: []string{"RoleId", "Tags", "AssumeRolePolicyDocument"},
222+
PreResourceResolver: "getRole",
222223
ExtraColumns: []codegen.ColumnDefinition{
223224
{
224225
Name: "account_id",
@@ -312,9 +313,10 @@ func IAMResources() []*Resource {
312313
},
313314
},
314315
{
315-
SubService: "users",
316-
Struct: &types.User{},
317-
SkipFields: []string{"Arn", "AccountId", "UserId", "Tags"},
316+
SubService: "users",
317+
Struct: &types.User{},
318+
SkipFields: []string{"Arn", "AccountId", "UserId", "Tags"},
319+
PreResourceResolver: "getUser",
318320
ExtraColumns: []codegen.ColumnDefinition{
319321
{
320322
Name: "arn",

‎plugins/source/aws/codegen/recipes/kinesis.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import (
99
func KinesisResources() []*Resource {
1010
resources := []*Resource{
1111
{
12-
SubService: "streams",
13-
Struct: &types.StreamDescriptionSummary{},
14-
SkipFields: []string{"StreamARN"},
12+
SubService: "streams",
13+
Struct: &types.StreamDescriptionSummary{},
14+
SkipFields: []string{"StreamARN"},
15+
PreResourceResolver: "getStream",
1516
ExtraColumns: append(
1617
defaultRegionalColumns,
1718
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/resourcegroups.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@ func ResourceGroupsResources() []*Resource {
1313
resources := []*Resource{
1414

1515
{
16-
SubService: "resource_groups",
17-
Struct: &models.ResourceGroupWrapper{},
18-
SkipFields: []string{},
16+
SubService: "resource_groups",
17+
Struct: &models.ResourceGroupWrapper{},
18+
SkipFields: []string{},
19+
PreResourceResolver: "getResourceGroup",
1920
ExtraColumns: append(
2021
defaultRegionalColumns,
2122
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/sagemaker.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,10 @@ func SagemakerResources() []*Resource {
7979
},
8080

8181
{
82-
SubService: "training_jobs",
83-
Struct: &sagemaker.DescribeTrainingJobOutput{},
84-
SkipFields: []string{"TrainingJobArn"},
82+
SubService: "training_jobs",
83+
Struct: &sagemaker.DescribeTrainingJobOutput{},
84+
SkipFields: []string{"TrainingJobArn"},
85+
PreResourceResolver: "getTrainingJob",
8586
ExtraColumns: append(
8687
defaultRegionalColumns,
8788
[]codegen.ColumnDefinition{

‎plugins/source/aws/codegen/recipes/sns.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,10 @@ func SNSResources() []*Resource {
1010
resources := []*Resource{
1111

1212
{
13-
SubService: "subscriptions",
14-
Struct: &models.Subscription{},
15-
SkipFields: []string{"SubscriptionArn", "DeliveryPolicy", "EffectiveDeliveryPolicy", "FilterPolicy", "RedrivePolicy"},
13+
SubService: "subscriptions",
14+
Struct: &models.Subscription{},
15+
SkipFields: []string{"SubscriptionArn", "DeliveryPolicy", "EffectiveDeliveryPolicy", "FilterPolicy", "RedrivePolicy"},
16+
PreResourceResolver: "getSnsSubscription",
1617
ExtraColumns: append(
1718
defaultRegionalColumns,
1819
[]codegen.ColumnDefinition{

0 commit comments

Comments
 (0)