Skip to content

Commit 5da8a1f

Browse files
amanenkamanenk
authored
fix!: Store some JSON columns as jsonb not as a text (#2463)
<!-- 🎉 Thank you for making CloudQuery awesome by submitting a PR 🎉 --> #### Summary closes #2430 <!-- Explain what problem this PR addresses --> <!-- Use the following steps to ensure your PR is ready to be reviewed - [ ] Read the [contribution guidelines](../blob/main/CONTRIBUTING.md) 🧑‍🎓 - [ ] Test locally on your own infrastructure - [ ] Run `go fmt` to format your code 🖊 - [ ] Lint your changes via `golangci-lint run` 🚨 (install golangci-lint [here](https://golangci-lint.run/usage/install/#local-installation)) - [ ] Update or add tests 🧪 - [ ] Ensure the status checks below are successful ✅ --->
1 parent 27dd558 commit 5da8a1f

24 files changed

+218
-77
lines changed

‎plugins/source/aws/client/resolvers.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package client
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
67
"reflect"
78
"time"
@@ -131,3 +132,34 @@ func SliceJsonResolver(path, keyPath, valuePath string) schema.ColumnResolver {
131132
return r.Set(c.Name, j)
132133
}
133134
}
135+
136+
// MarshaledJsonResolver resolves a json string into a map[string]interface{}.
137+
func MarshaledJsonResolver(path string) schema.ColumnResolver {
138+
return func(_ context.Context, meta schema.ClientMeta, r *schema.Resource, c schema.Column) error {
139+
var j map[string]interface{}
140+
field := funk.Get(r.Item, path, funk.WithAllowZero())
141+
142+
var val reflect.Value
143+
if reflect.TypeOf(field).Kind() == reflect.Ptr {
144+
s := reflect.ValueOf(field)
145+
if s.IsNil() {
146+
return nil
147+
}
148+
val = s.Elem()
149+
} else {
150+
val = reflect.ValueOf(field)
151+
}
152+
var err error
153+
j = make(map[string]interface{})
154+
switch val.Kind() {
155+
case reflect.String:
156+
err = json.Unmarshal([]byte(val.String()), &j)
157+
case reflect.Slice, reflect.Array:
158+
err = json.Unmarshal(val.Bytes(), &j)
159+
}
160+
if err != nil {
161+
return errors.WithStack(err)
162+
}
163+
return r.Set(c.Name, j)
164+
}
165+
}

‎plugins/source/aws/client/resolvers_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,3 +146,58 @@ func TestResolveSliceJson(t *testing.T) {
146146
assert.Equal(t, tc.ExpectedData, r.Get(ta.Columns[0].Name))
147147
}
148148
}
149+
150+
var jsonString = "{\"k1\":\"v1\"}"
151+
var jsonBytes = []byte(jsonString)
152+
153+
func TestResolveStringJson(t *testing.T) {
154+
cases := []struct {
155+
InputItem interface{}
156+
ExpectedData map[string]interface{}
157+
Path string
158+
}{
159+
{
160+
InputItem: struct {
161+
Json string
162+
}{Json: jsonString},
163+
ExpectedData: map[string]interface{}{"k1": "v1"},
164+
Path: "Json",
165+
},
166+
{
167+
InputItem: struct {
168+
Json *string
169+
}{Json: &jsonString},
170+
ExpectedData: map[string]interface{}{"k1": "v1"},
171+
Path: "Json",
172+
},
173+
{
174+
InputItem: struct {
175+
Json []byte
176+
}{Json: jsonBytes},
177+
ExpectedData: map[string]interface{}{"k1": "v1"},
178+
Path: "Json",
179+
},
180+
{
181+
InputItem: struct {
182+
Json *[]byte
183+
}{Json: &jsonBytes},
184+
ExpectedData: map[string]interface{}{"k1": "v1"},
185+
Path: "Json",
186+
},
187+
}
188+
189+
for _, tc := range cases {
190+
ta := &schema.Table{
191+
Columns: []schema.Column{
192+
{
193+
Name: "json",
194+
Type: schema.TypeJSON,
195+
},
196+
},
197+
}
198+
r := schema.NewResourceData(ta, nil, tc.InputItem)
199+
err := MarshaledJsonResolver(tc.Path)(context.Background(), nil, r, ta.Columns[0])
200+
assert.NoError(t, err)
201+
assert.Equal(t, tc.ExpectedData, r.Get(ta.Columns[0].Name))
202+
}
203+
}

‎plugins/source/aws/codegen/recipes/codepipeline.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func CodePipelineResources() []*Resource {
4242
},
4343
{
4444
Name: "tags",
45-
Type: schema.TypeString,
45+
Type: schema.TypeJSON,
4646
Resolver: `resolvePipelineTags`,
4747
},
4848
}...),

‎plugins/source/aws/codegen/recipes/dax.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func DaxResources() []*Resource {
2424
},
2525
{
2626
Name: "tags",
27-
Type: schema.TypeString,
27+
Type: schema.TypeJSON,
2828
Resolver: `resolveClusterTags`,
2929
},
3030
}...),

‎plugins/source/aws/codegen/recipes/ecr.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func ECRResources() []*Resource {
2424
},
2525
{
2626
Name: "tags",
27-
Type: schema.TypeString,
27+
Type: schema.TypeJSON,
2828
Resolver: `resolveRepositoryTags`,
2929
},
3030
}...),

‎plugins/source/aws/codegen/recipes/route53.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ func Route53Resources() []*Resource {
174174
{
175175
SubService: "traffic_policy_versions",
176176
Struct: &types.TrafficPolicy{},
177-
SkipFields: []string{"Version", "Id"},
177+
SkipFields: []string{"Version", "Id", "Document"},
178178
ExtraColumns: append(
179179
defaultAccountColumns,
180180
[]codegen.ColumnDefinition{
@@ -196,6 +196,11 @@ func Route53Resources() []*Resource {
196196
Resolver: `schema.PathResolver("Version")`,
197197
Options: schema.ColumnCreationOptions{PrimaryKey: true},
198198
},
199+
{
200+
Name: "document",
201+
Type: schema.TypeJSON,
202+
Resolver: `client.MarshaledJsonResolver("Document")`,
203+
},
199204
}...),
200205
},
201206
}

‎plugins/source/aws/codegen/recipes/sns.go‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ func SNSResources() []*Resource {
1212
{
1313
SubService: "subscriptions",
1414
Struct: &models.Subscription{},
15-
SkipFields: []string{"SubscriptionArn"},
15+
SkipFields: []string{"SubscriptionArn", "DeliveryPolicy", "EffectiveDeliveryPolicy", "FilterPolicy", "RedrivePolicy"},
1616
ExtraColumns: append(
1717
defaultRegionalColumns,
1818
[]codegen.ColumnDefinition{
@@ -22,13 +22,33 @@ func SNSResources() []*Resource {
2222
Resolver: `schema.PathResolver("SubscriptionArn")`,
2323
Options: schema.ColumnCreationOptions{PrimaryKey: true},
2424
},
25+
{
26+
Name: "delivery_policy",
27+
Type: schema.TypeJSON,
28+
Resolver: `client.MarshaledJsonResolver("DeliveryPolicy")`,
29+
},
30+
{
31+
Name: "effective_delivery_policy",
32+
Type: schema.TypeJSON,
33+
Resolver: `client.MarshaledJsonResolver("EffectiveDeliveryPolicy")`,
34+
},
35+
{
36+
Name: "filter_policy",
37+
Type: schema.TypeJSON,
38+
Resolver: `client.MarshaledJsonResolver("FilterPolicy")`,
39+
},
40+
{
41+
Name: "redrive_policy",
42+
Type: schema.TypeJSON,
43+
Resolver: `client.MarshaledJsonResolver("RedrivePolicy")`,
44+
},
2545
}...),
2646
},
2747

2848
{
2949
SubService: "topics",
3050
Struct: &models.Topic{},
31-
SkipFields: []string{"Arn"},
51+
SkipFields: []string{"Arn", "Policy", "EffectiveDeliveryPolicy", "DeliveryPolicy"},
3252
ExtraColumns: append(
3353
defaultRegionalColumns,
3454
[]codegen.ColumnDefinition{
@@ -43,6 +63,21 @@ func SNSResources() []*Resource {
4363
Type: schema.TypeJSON,
4464
Resolver: `resolveSnsTopicTags`,
4565
},
66+
{
67+
Name: "delivery_policy",
68+
Type: schema.TypeJSON,
69+
Resolver: `client.MarshaledJsonResolver("DeliveryPolicy")`,
70+
},
71+
{
72+
Name: "policy",
73+
Type: schema.TypeJSON,
74+
Resolver: `client.MarshaledJsonResolver("Policy")`,
75+
},
76+
{
77+
Name: "effective_delivery_policy",
78+
Type: schema.TypeJSON,
79+
Resolver: `client.MarshaledJsonResolver("EffectiveDeliveryPolicy")`,
80+
},
4681
}...),
4782
},
4883
}

‎plugins/source/aws/codegen/recipes/sqs.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,10 @@ import (
88

99
func SQSResources() []*Resource {
1010
resources := []*Resource{
11-
1211
{
1312
SubService: "queues",
1413
Struct: &models.Queue{},
15-
SkipFields: []string{"Arn"},
14+
SkipFields: []string{"Arn", "Policy", "RedriveAllowPolicy", "RedrivePolicy"},
1615
ExtraColumns: append(
1716
defaultRegionalColumns,
1817
[]codegen.ColumnDefinition{
@@ -27,6 +26,21 @@ func SQSResources() []*Resource {
2726
Type: schema.TypeJSON,
2827
Resolver: `resolveSqsQueueTags`,
2928
},
29+
{
30+
Name: "policy",
31+
Type: schema.TypeJSON,
32+
Resolver: `client.MarshaledJsonResolver("Policy")`,
33+
},
34+
{
35+
Name: "redrive_policy",
36+
Type: schema.TypeJSON,
37+
Resolver: `client.MarshaledJsonResolver("RedrivePolicy")`,
38+
},
39+
{
40+
Name: "redrive_allow_policy",
41+
Type: schema.TypeJSON,
42+
Resolver: `client.MarshaledJsonResolver("RedriveAllowPolicy")`,
43+
},
3044
}...),
3145
},
3246
}

‎plugins/source/aws/docs/tables/aws_codepipeline_pipelines.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@ The primary key for this table is **arn**.
1515
|account_id|String|
1616
|region|String|
1717
|arn (PK)|String|
18-
|tags|String|
18+
|tags|JSON|
1919
|metadata|JSON|
2020
|pipeline|JSON|

‎plugins/source/aws/docs/tables/aws_dax_clusters.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ The primary key for this table is **arn**.
1515
|account_id|String|
1616
|region|String|
1717
|arn (PK)|String|
18-
|tags|String|
18+
|tags|JSON|
1919
|active_nodes|Int|
2020
|cluster_discovery_endpoint|JSON|
2121
|cluster_endpoint_encryption_type|String|

0 commit comments

Comments
 (0)