Skip to content

Commit 7908fd4

Browse files
hermanschaafhermanschaaf
authored
feat(aws)!: Support recursive listing of AWS orgs and skipping of OUs and accounts (#5870)
This changes the behavior of OUs to also traverse nested (child) OUs, if any. It also adds two new `org`-level config options: `skip_organization_units` and `skip_accounts`. - Closes #5029 - Replaces #5864 BEGIN_COMMIT_OVERRIDE feat(aws): Support recursive listing of AWS orgs and skipping of OUs and accounts BREAKING CHANGE: Child OUs will now also be recursively traversed when specifying OUs in AWS org mode BREAKING CHANGE: Account IDs are now validated to be exactly 12 digits BREAKING CHANGE: Organizational Unit (OU) IDs are now validated to match either `ou-` or `r-` formats END_COMMIT_OVERRIDE
1 parent fdfd172 commit 7908fd4

7 files changed

Lines changed: 456 additions & 96 deletions

File tree

‎plugins/source/aws/client/client.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -307,11 +307,14 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source) (s
307307
return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
308308
}
309309

310+
err = awsConfig.Validate()
311+
if err != nil {
312+
return nil, fmt.Errorf("spec validation failed: %w", err)
313+
}
314+
310315
client := NewAwsClient(logger)
311316
var adminAccountSts AssumeRoleAPIClient
312-
if awsConfig.Organization != nil && len(awsConfig.Accounts) > 0 {
313-
return nil, errors.New("specifying accounts via both the Accounts and Org properties is not supported. To achieve both, use multiple source configurations")
314-
}
317+
315318
if awsConfig.Organization != nil {
316319
var err error
317320
awsConfig.Accounts, adminAccountSts, err = loadOrgAccounts(ctx, logger, &awsConfig)

‎plugins/source/aws/client/organizations.go‎

Lines changed: 68 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package client
22

33
import (
44
"context"
5+
56
"github.com/cloudquery/cloudquery/plugins/source/aws/client/services"
7+
"github.com/thoas/go-funk"
68

79
"github.com/aws/aws-sdk-go-v2/aws"
810
"github.com/aws/aws-sdk-go-v2/aws/arn"
@@ -44,20 +46,28 @@ func loadAccounts(ctx context.Context, awsConfig *Spec, accountsApi services.Org
4446
var rawAccounts []orgTypes.Account
4547
var err error
4648
if len(awsConfig.Organization.OrganizationUnits) > 0 {
47-
rawAccounts, err = getOUAccounts(ctx, accountsApi, awsConfig.Organization.OrganizationUnits)
49+
rawAccounts, err = getOUAccounts(ctx, accountsApi, awsConfig.Organization)
4850
} else {
49-
rawAccounts, err = getAllAccounts(ctx, accountsApi)
51+
rawAccounts, err = getAllAccounts(ctx, accountsApi, awsConfig.Organization)
5052
}
5153

5254
if err != nil {
5355
return []Account{}, err
5456
}
57+
seen := map[string]struct{}{}
5558
accounts := make([]Account, 0)
5659
for _, account := range rawAccounts {
5760
// Only load Active accounts
58-
if account.Status != orgTypes.AccountStatusActive {
61+
if account.Status != orgTypes.AccountStatusActive || account.Id == nil {
62+
continue
63+
}
64+
65+
// Skip duplicates
66+
if _, found := seen[*account.Id]; found {
5967
continue
6068
}
69+
seen[*account.Id] = struct{}{}
70+
6171
roleArn := arn.ARN{
6272
Partition: "aws",
6373
Service: "iam",
@@ -83,46 +93,78 @@ func loadAccounts(ctx context.Context, awsConfig *Spec, accountsApi services.Org
8393
}
8494

8595
// Get Accounts for specific Organizational Units
86-
func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, ous []string) ([]orgTypes.Account, error) {
96+
func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *AwsOrg) ([]orgTypes.Account, error) {
97+
q := awsOrg.OrganizationUnits
98+
var ou string
8799
var rawAccounts []orgTypes.Account
100+
seenOUs := map[string]struct{}{}
101+
for len(q) > 0 {
102+
ou, q = q[0], q[1:]
103+
104+
// Skip duplicates to avoid making duplicate API calls
105+
if _, found := seenOUs[ou]; found {
106+
continue
107+
}
108+
seenOUs[ou] = struct{}{}
109+
110+
// Skip any OUs that user has asked to skip
111+
if funk.ContainsString(awsOrg.SkipOrganizationalUnits, ou) {
112+
continue
113+
}
88114

89-
for _, ou := range ous {
90-
var paginationToken *string
91-
for {
92-
resp, err := accountsApi.ListAccountsForParent(ctx, &organizations.ListAccountsForParentInput{
93-
NextToken: paginationToken,
94-
ParentId: aws.String(ou),
95-
})
115+
// get accounts directly under this OU
116+
accountsPaginator := organizations.NewListAccountsForParentPaginator(accountsApi, &organizations.ListAccountsForParentInput{
117+
ParentId: aws.String(ou),
118+
})
119+
for accountsPaginator.HasMorePages() {
120+
output, err := accountsPaginator.NextPage(ctx)
96121
if err != nil {
97122
return nil, err
98123
}
99-
rawAccounts = append(rawAccounts, resp.Accounts...)
100-
if resp.NextToken == nil {
101-
break
124+
for _, account := range output.Accounts {
125+
// Skip any accounts that user has asked to skip
126+
if funk.ContainsString(awsOrg.SkipMemberAccounts, *account.Id) {
127+
continue
128+
}
129+
rawAccounts = append(rawAccounts, account)
130+
}
131+
}
132+
133+
// get OUs directly under this OU, and add them to the queue
134+
ouPaginator := organizations.NewListChildrenPaginator(accountsApi, &organizations.ListChildrenInput{
135+
ChildType: orgTypes.ChildTypeOrganizationalUnit,
136+
ParentId: aws.String(ou),
137+
})
138+
for ouPaginator.HasMorePages() {
139+
output, err := ouPaginator.NextPage(ctx)
140+
if err != nil {
141+
return nil, err
142+
}
143+
for _, child := range output.Children {
144+
q = append(q, *child.Id)
102145
}
103-
paginationToken = resp.NextToken
104146
}
105147
}
148+
106149
return rawAccounts, nil
107150
}
108151

109152
// Get All accounts in a specific organization
110-
func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient) ([]orgTypes.Account, error) {
153+
func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *AwsOrg) ([]orgTypes.Account, error) {
111154
var rawAccounts []orgTypes.Account
112-
var paginationToken *string
113-
114-
for {
115-
resp, err := accountsApi.ListAccounts(ctx, &organizations.ListAccountsInput{
116-
NextToken: paginationToken,
117-
})
155+
accountsPaginator := organizations.NewListAccountsPaginator(accountsApi, &organizations.ListAccountsInput{})
156+
for accountsPaginator.HasMorePages() {
157+
output, err := accountsPaginator.NextPage(ctx)
118158
if err != nil {
119159
return nil, err
120160
}
121-
rawAccounts = append(rawAccounts, resp.Accounts...)
122-
if resp.NextToken == nil {
123-
break
161+
for _, account := range output.Accounts {
162+
// Skip any accounts that user has asked to skip
163+
if funk.ContainsString(org.SkipMemberAccounts, *account.Id) {
164+
continue
165+
}
166+
rawAccounts = append(rawAccounts, account)
124167
}
125-
paginationToken = resp.NextToken
126168
}
127169
return rawAccounts, nil
128170
}

0 commit comments

Comments
 (0)