Skip to content

Commit a8a08c3

Browse files
disqerezrokah
disq
and
erezrokah
authored
feat(aws): Managed Workflows for Apache Airflow (MWAA) environments (#3431)
Implements #2300 Co-authored-by: Kemal Hadimli <disq@users.noreply.github.com> Co-authored-by: Erez Rokah <erezrokah@users.noreply.github.com>
1 parent 191a203 commit a8a08c3

File tree

14 files changed

+439
-1
lines changed

14 files changed

+439
-1
lines changed

‎plugins/source/aws/client/mocks/mwaa.go‎

Lines changed: 96 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/aws/client/services.go‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/aws/client/services/mwaa.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/aws/codegen/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
package main
22

33
import (
4+
"log"
5+
46
"github.com/cloudquery/cloudquery/plugins/source/aws/codegen/recipes"
57
"github.com/cloudquery/cloudquery/plugins/source/aws/codegen/services"
68
"github.com/cloudquery/cloudquery/plugins/source/aws/codegen/tables"
7-
"log"
89
)
910

1011
func generateResources() ([]*recipes.Resource, error) {
@@ -62,6 +63,7 @@ func generateResources() ([]*recipes.Resource, error) {
6263
resources = append(resources, recipes.LambdaResources()...)
6364
resources = append(resources, recipes.LightsailResources()...)
6465
resources = append(resources, recipes.MQResources()...)
66+
resources = append(resources, recipes.MWAAResources()...)
6567
resources = append(resources, recipes.NeptuneResources()...)
6668
resources = append(resources, recipes.OrganizationsResources()...)
6769
resources = append(resources, recipes.QLDBResources()...)

‎plugins/source/aws/codegen/recipes/mwaa.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package recipes
2+
3+
import (
4+
"github.com/aws/aws-sdk-go-v2/service/mwaa/types"
5+
"github.com/cloudquery/plugin-sdk/codegen"
6+
"github.com/cloudquery/plugin-sdk/schema"
7+
)
8+
9+
func MWAAResources() []*Resource {
10+
resources := []*Resource{
11+
{
12+
SubService: "environments",
13+
Description: "https://docs.aws.amazon.com/mwaa/latest/API/API_Environment.html",
14+
Struct: &types.Environment{},
15+
SkipFields: []string{"Arn"},
16+
PreResourceResolver: "getEnvironment",
17+
ExtraColumns: append(
18+
defaultRegionalColumns,
19+
[]codegen.ColumnDefinition{
20+
{
21+
Name: "arn",
22+
Type: schema.TypeString,
23+
Resolver: `schema.PathResolver("Arn")`,
24+
Options: schema.ColumnCreationOptions{PrimaryKey: true},
25+
},
26+
}...),
27+
},
28+
}
29+
30+
// set default values
31+
for _, r := range resources {
32+
r.Service = "mwaa"
33+
r.Multiplex = `client.ServiceAccountRegionMultiplexer("airflow")`
34+
}
35+
return resources
36+
}

‎plugins/source/aws/codegen/services/clients.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import (
5555
"github.com/aws/aws-sdk-go-v2/service/lambda"
5656
"github.com/aws/aws-sdk-go-v2/service/lightsail"
5757
"github.com/aws/aws-sdk-go-v2/service/mq"
58+
"github.com/aws/aws-sdk-go-v2/service/mwaa"
5859
"github.com/aws/aws-sdk-go-v2/service/neptune"
5960
"github.com/aws/aws-sdk-go-v2/service/organizations"
6061
"github.com/aws/aws-sdk-go-v2/service/qldb"
@@ -137,6 +138,7 @@ var clients = []interface{}{
137138
&lambda.Client{},
138139
&lightsail.Client{},
139140
&mq.Client{},
141+
&mwaa.Client{},
140142
&neptune.Client{},
141143
&organizations.Client{},
142144
&qldb.Client{},

‎plugins/source/aws/docs/tables/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@
278278
| [aws_mq_brokers](aws_mq_brokers.md) |
279279
| ↳ [aws_mq_broker_configurations](aws_mq_broker_configurations.md) |
280280
| ↳ [aws_mq_broker_users](aws_mq_broker_users.md) |
281+
| [aws_mwaa_environments](aws_mwaa_environments.md) |
281282
| [aws_neptune_clusters](aws_neptune_clusters.md) |
282283
| [aws_neptune_db_parameter_groups](aws_neptune_db_parameter_groups.md) |
283284
| ↳ [aws_neptune_db_parameter_group_db_parameters](aws_neptune_db_parameter_group_db_parameters.md) |

‎plugins/source/aws/docs/tables/aws_mwaa_environments.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Table: aws_mwaa_environments
2+
3+
https://docs.aws.amazon.com/mwaa/latest/API/API_Environment.html
4+
5+
The primary key for this table is **arn**.
6+
7+
8+
## Columns
9+
| Name | Type |
10+
| ------------- | ------------- |
11+
|_cq_source_name|String|
12+
|_cq_sync_time|Timestamp|
13+
|_cq_id|UUID|
14+
|_cq_parent_id|UUID|
15+
|account_id|String|
16+
|region|String|
17+
|arn (PK)|String|
18+
|airflow_configuration_options|JSON|
19+
|airflow_version|String|
20+
|created_at|Timestamp|
21+
|dag_s3_path|String|
22+
|environment_class|String|
23+
|execution_role_arn|String|
24+
|kms_key|String|
25+
|last_update|JSON|
26+
|logging_configuration|JSON|
27+
|max_workers|Int|
28+
|min_workers|Int|
29+
|name|String|
30+
|network_configuration|JSON|
31+
|plugins_s3_object_version|String|
32+
|plugins_s3_path|String|
33+
|requirements_s3_object_version|String|
34+
|requirements_s3_path|String|
35+
|schedulers|Int|
36+
|service_role_arn|String|
37+
|source_bucket_arn|String|
38+
|status|String|
39+
|tags|JSON|
40+
|webserver_access_mode|String|
41+
|webserver_url|String|
42+
|weekly_maintenance_window_start|String|

‎plugins/source/aws/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ require (
9898
require (
9999
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 // indirect
100100
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 // indirect
101+
github.com/aws/aws-sdk-go-v2/service/mwaa v1.13.12 // indirect
101102
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
102103
github.com/bitly/go-simplejson v0.5.0 // indirect
103104
github.com/getsentry/sentry-go v0.14.0 // indirect

‎plugins/source/aws/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ github.com/aws/aws-sdk-go-v2/service/lightsail v1.23.2 h1:taL77V/XEcokFi29cmG9aF
138138
github.com/aws/aws-sdk-go-v2/service/lightsail v1.23.2/go.mod h1:LYCAGTJ4XgNZAESKxLC9L12F7IIQbJHccVxanY8DYzo=
139139
github.com/aws/aws-sdk-go-v2/service/mq v1.13.15 h1:K0UbxNfaPHoIXo0T4L4jrbE+TKcVPYp55nEnMLxVRPk=
140140
github.com/aws/aws-sdk-go-v2/service/mq v1.13.15/go.mod h1:ycghPMXYItx5Y74iehFgGwDNUMXdq0xCxLaYC5uYZO0=
141+
github.com/aws/aws-sdk-go-v2/service/mwaa v1.13.12 h1:9piNP7cXEY4/kyb9jp2mZmApADQ4xSPmybNd/N+BFCc=
142+
github.com/aws/aws-sdk-go-v2/service/mwaa v1.13.12/go.mod h1:fU/hThlyHRbtVU2K0R1iWi65xByYVi+Rcxti3pr4zdQ=
141143
github.com/aws/aws-sdk-go-v2/service/neptune v1.18.0 h1:KjuTmFGJsFIkdrwb3Mn5vifuZQ7po2LuEoiVizMztgg=
142144
github.com/aws/aws-sdk-go-v2/service/neptune v1.18.0/go.mod h1:YW+k1DAbNfgK2aS3AGPYSbNWCfsclfzEf4ZsNMGFdik=
143145
github.com/aws/aws-sdk-go-v2/service/organizations v1.16.15 h1:DKPB04iAh04HwzriUgKlnRYfrpQzWkbjRdvnePO1glM=

0 commit comments

Comments
 (0)