Skip to content

Commit 00f71ed

Browse files
hermanschaafhermanschaaf
authored
fix: Fix date parsing in Credential Report CSVs (v2) (#1902)
This fixes a couple of issues with credential reports: - Credential Report CSVs can have values like `N/A` and `not_supported`, which previously failed to parse as dates, but now will result in nil columns - Some columns were strings when they should have been dates This was quite hard to fix, as it required creation of a custom type to the unmarshal CSV date, as well as a custom resolver to convert this to a normal timestamp. I also added a custom test to test different scenarios, including one where we expect certain columns to be nil, but want to check that date parsing still works.
1 parent 3469f20 commit 00f71ed

4 files changed

Lines changed: 214 additions & 92 deletions

File tree

‎plugins/source/aws/codegen/recipes/iam.go‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,19 @@ func IAMResources() []*Resource {
2929
{
3030
SubService: "credential_reports",
3131
Struct: &iamService.CredentialReportEntry{},
32-
SkipFields: []string{"Arn", "UserCreationTime"},
32+
SkipFields: []string{
33+
"Arn",
34+
"UserCreationTime",
35+
"PasswordLastChanged",
36+
"PasswordNextRotation",
37+
"AccessKey1LastRotated",
38+
"AccessKey2LastRotated",
39+
"Cert1LastRotated",
40+
"Cert2LastRotated",
41+
"AccessKey1LastUsedDate",
42+
"AccessKey2LastUsedDate",
43+
"PasswordLastUsed",
44+
},
3345
ExtraColumns: []codegen.ColumnDefinition{
3446
{
3547
Name: "arn",
@@ -40,9 +52,54 @@ func IAMResources() []*Resource {
4052
{
4153
Name: "user_creation_time",
4254
Type: schema.TypeTimestamp,
43-
Resolver: `schema.PathResolver("UserCreationTime")`,
55+
Resolver: `timestampPathResolver("UserCreationTime")`,
4456
Options: schema.ColumnCreationOptions{PrimaryKey: true},
4557
},
58+
{
59+
Name: "password_last_changed",
60+
Type: schema.TypeTimestamp,
61+
Resolver: `timestampPathResolver("PasswordLastChanged")`,
62+
},
63+
{
64+
Name: "password_next_rotation",
65+
Type: schema.TypeTimestamp,
66+
Resolver: `timestampPathResolver("PasswordNextRotation")`,
67+
},
68+
{
69+
Name: "access_key_1_last_rotated",
70+
Type: schema.TypeTimestamp,
71+
Resolver: `timestampPathResolver("AccessKey1LastRotated")`,
72+
},
73+
{
74+
Name: "access_key_2_last_rotated",
75+
Type: schema.TypeTimestamp,
76+
Resolver: `timestampPathResolver("AccessKey2LastRotated")`,
77+
},
78+
{
79+
Name: "cert_1_last_rotated",
80+
Type: schema.TypeTimestamp,
81+
Resolver: `timestampPathResolver("Cert1LastRotated")`,
82+
},
83+
{
84+
Name: "cert_2_last_rotated",
85+
Type: schema.TypeTimestamp,
86+
Resolver: `timestampPathResolver("Cert2LastRotated")`,
87+
},
88+
{
89+
Name: "access_key_1_last_used_date",
90+
Type: schema.TypeTimestamp,
91+
Resolver: `timestampPathResolver("AccessKey1LastUsedDate")`,
92+
},
93+
{
94+
Name: "access_key_2_last_used_date",
95+
Type: schema.TypeTimestamp,
96+
Resolver: `timestampPathResolver("AccessKey2LastUsedDate")`,
97+
},
98+
{
99+
Name: "password_last_used",
100+
Type: schema.TypeTimestamp,
101+
Resolver: `timestampPathResolver("PasswordLastUsed")`,
102+
},
46103
},
47104
Relations: []string{},
48105
},

‎plugins/source/aws/resources/services/iam/credential_reports.go‎

Lines changed: 46 additions & 46 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/aws/resources/services/iam/credential_reports_fetch.go‎

Lines changed: 48 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,50 @@ import (
1010
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
1111
"github.com/cloudquery/plugin-sdk/schema"
1212
"github.com/gocarina/gocsv"
13+
"github.com/thoas/go-funk"
1314
)
1415

1516
type CredentialReportEntry struct {
16-
User string `csv:"user"`
17-
Arn string `csv:"arn"`
18-
UserCreationTime time.Time `csv:"user_creation_time"`
19-
PasswordStatus string `csv:"password_enabled"`
20-
PasswordLastChanged string `csv:"password_last_changed"`
21-
PasswordNextRotation string `csv:"password_next_rotation"`
22-
MfaActive bool `csv:"mfa_active"`
23-
AccessKey1Active bool `csv:"access_key_1_active"`
24-
AccessKey2Active bool `csv:"access_key_2_active"`
25-
AccessKey1LastRotated string `csv:"access_key_1_last_rotated"`
26-
AccessKey2LastRotated string `csv:"access_key_2_last_rotated"`
27-
Cert1Active bool `csv:"cert_1_active"`
28-
Cert2Active bool `csv:"cert_2_active"`
29-
Cert1LastRotated string `csv:"cert_1_last_rotated"`
30-
Cert2LastRotated string `csv:"cert_2_last_rotated"`
31-
AccessKey1LastUsedDate time.Time `csv:"access_key_1_last_used_date"`
32-
AccessKey1LastUsedRegion string `csv:"access_key_1_last_used_region"`
33-
AccessKey1LastUsedService string `csv:"access_key_1_last_used_service"`
34-
AccessKey2LastUsedDate time.Time `csv:"access_key_2_last_used_date"`
35-
AccessKey2LastUsedRegion string `csv:"access_key_2_last_used_region"`
36-
AccessKey2LastUsedService string `csv:"access_key_2_last_used_service"`
37-
PasswordLastUsed string `csv:"password_last_used"`
17+
User string `csv:"user"`
18+
Arn string `csv:"arn"`
19+
UserCreationTime DateTime `csv:"user_creation_time"`
20+
PasswordStatus string `csv:"password_enabled"`
21+
PasswordLastChanged DateTime `csv:"password_last_changed"`
22+
PasswordNextRotation DateTime `csv:"password_next_rotation"`
23+
MfaActive bool `csv:"mfa_active"`
24+
AccessKey1Active bool `csv:"access_key_1_active"`
25+
AccessKey2Active bool `csv:"access_key_2_active"`
26+
AccessKey1LastRotated DateTime `csv:"access_key_1_last_rotated"`
27+
AccessKey2LastRotated DateTime `csv:"access_key_2_last_rotated"`
28+
Cert1Active bool `csv:"cert_1_active"`
29+
Cert2Active bool `csv:"cert_2_active"`
30+
Cert1LastRotated DateTime `csv:"cert_1_last_rotated"`
31+
Cert2LastRotated DateTime `csv:"cert_2_last_rotated"`
32+
AccessKey1LastUsedDate DateTime `csv:"access_key_1_last_used_date"`
33+
AccessKey1LastUsedRegion string `csv:"access_key_1_last_used_region"`
34+
AccessKey1LastUsedService string `csv:"access_key_1_last_used_service"`
35+
AccessKey2LastUsedDate DateTime `csv:"access_key_2_last_used_date"`
36+
AccessKey2LastUsedRegion string `csv:"access_key_2_last_used_region"`
37+
AccessKey2LastUsedService string `csv:"access_key_2_last_used_service"`
38+
PasswordLastUsed DateTime `csv:"password_last_used"`
39+
}
40+
41+
type DateTime struct {
42+
*time.Time
43+
}
44+
45+
func (d *DateTime) UnmarshalCSV(val string) (err error) {
46+
switch val {
47+
case "N/A", "not_supported":
48+
d.Time = nil
49+
return nil
50+
}
51+
t, err := time.Parse(time.RFC3339, val)
52+
if err != nil {
53+
return err
54+
}
55+
d.Time = &t
56+
return nil
3857
}
3958

4059
func fetchIamCredentialReports(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan<- interface{}) error {
@@ -83,3 +102,10 @@ func fetchIamCredentialReports(ctx context.Context, meta schema.ClientMeta, _ *s
83102
}
84103
}
85104
}
105+
func timestampPathResolver(path string) schema.ColumnResolver {
106+
return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
107+
t := funk.Get(r.Item, path, funk.WithAllowZero())
108+
dt := t.(DateTime)
109+
return r.Set(c.Name, dt.Time)
110+
}
111+
}

0 commit comments

Comments
 (0)