Skip to content

Commit e5774fa

Browse files
irmatovirmatov
authored
feat: Add fsx data repo tasks (#1279)
1 parent a740719 commit e5774fa

17 files changed

+406
-118
lines changed

‎plugins/source/aws/client/mocks/fsx.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/aws/client/services.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,7 @@ type FirehoseClient interface {
441441
//go:generate mockgen -package=mocks -destination=./mocks/fsx.go . FsxClient
442442
type FsxClient interface {
443443
DescribeBackups(ctx context.Context, params *fsx.DescribeBackupsInput, optFns ...func(*fsx.Options)) (*fsx.DescribeBackupsOutput, error)
444+
DescribeDataRepositoryTasks(ctx context.Context, params *fsx.DescribeDataRepositoryTasksInput, optFns ...func(*fsx.Options)) (*fsx.DescribeDataRepositoryTasksOutput, error)
444445
DescribeFileSystems(ctx context.Context, params *fsx.DescribeFileSystemsInput, optFns ...func(*fsx.Options)) (*fsx.DescribeFileSystemsOutput, error)
445446
DescribeSnapshots(ctx context.Context, params *fsx.DescribeSnapshotsInput, optFns ...func(*fsx.Options)) (*fsx.DescribeSnapshotsOutput, error)
446447
}

‎plugins/source/aws/docs/tables/aws_fsx_data_repo_tasks.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
2+
# Table: aws_fsx_data_repo_tasks
3+
A description of the data repository task
4+
## Columns
5+
| Name | Type | Description |
6+
| ------------- | ------------- | ----- |
7+
|account_id|text|The AWS Account ID of the resource|
8+
|region|text|The AWS Region of the resource|
9+
|creation_time|timestamp without time zone|The time that the resource was created, in seconds (since 1970-01-01T00:00:00Z), also known as Unix time|
10+
|file_system_id|text|The globally unique ID of the file system, assigned by Amazon FSx|
11+
|lifecycle|text|The lifecycle status of the data repository task, as follows: * PENDING - Amazon FSx has not started the task|
12+
|task_id|text|The system-generated, unique 17-digit ID of the data repository task|
13+
|type|text|The type of data repository task|
14+
|end_time|timestamp without time zone|The time that Amazon FSx completed processing the task, populated after the task is complete|
15+
|failure_details_message|text|A detailed error message|
16+
|paths|text[]|An array of paths on the Amazon FSx for Lustre file system that specify the data for the data repository task to process|
17+
|report_enabled|boolean|Set Enabled to True to generate a CompletionReport when the task completes|
18+
|report_format|text|Required if Enabled is set to true|
19+
|report_path|text|Required if Enabled is set to true|
20+
|report_scope|text|Required if Enabled is set to true|
21+
|arn|text|The Amazon Resource Name (ARN) for a given resource|
22+
|start_time|timestamp without time zone|The time that Amazon FSx began processing the task|
23+
|status_failed_count|bigint|A running total of the number of files that the task failed to process|
24+
|status_last_updated_time|timestamp without time zone|The time at which the task status was last updated|
25+
|status_succeeded_count|bigint|A running total of the number of files that the task has successfully processed|
26+
|status_total_count|bigint|The total number of files that the task will process|
27+
|tags|jsonb|A list of Tag values, with a maximum of 50 elements|

‎plugins/source/aws/docs/tables/aws_fsx_filesystem_lustre_configuration.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11

22
# Table: aws_fsx_filesystem_lustre_configuration
3-
The configuration for the Amazon FSx for Lustre file system.
3+
The configuration for the Amazon FSx for Lustre file system
44
## Columns
55
| Name | Type | Description |
66
| ------------- | ------------- | ----- |
77
|filesystem_cq_id|uuid|Unique CloudQuery ID of aws_fsx_filesystems table (FK)|
88
|automatic_backup_retention_days|bigint|The number of days to retain automatic backups|
9-
|copy_tags_to_backups|boolean|A boolean flag indicating whether tags on the file system are copied to backups. If it's set to true, all tags on the file system are copied to all automatic backups and any user-initiated backups where the user doesn't specify any tags. If this value is true, and you specify one or more tags, only the specified tags are copied to backups|
9+
|copy_tags_to_backups|boolean|A boolean flag indicating whether tags on the file system are copied to backups|
1010
|daily_automatic_backup_start_time|text|A recurring daily time, in the format HH:MM|
1111
|data_compression_type|text|The data compression configuration for the file system|
1212
|data_repo_cfg_auto_import_policy|text|Describes the file system's linked S3 data repository's AutoImportPolicy|
13-
|data_repo_cfg_export_path|text|The export path to the Amazon S3 bucket (and prefix) that you are using to store new and changed Lustre file system files in S3.|
14-
|data_repo_cfg_failure_details_message|text|A detailed error message.|
13+
|data_repo_cfg_export_path|text|The export path to the Amazon S3 bucket (and prefix) that you are using to store new and changed Lustre file system files in S3|
14+
|data_repo_cfg_failure_details_message|text|A detailed error message|
1515
|data_repo_cfg_import_path|text|The import path to the Amazon S3 bucket (and optional prefix) that you're using as the data repository for your FSx for Lustre file system, for example s3://import-bucket/optional-prefix|
1616
|data_repo_cfg_imported_file_chunk_size|bigint|For files imported from a data repository, this value determines the stripe count and maximum amount of data per file (in MiB) stored on a single physical disk|
1717
|data_repo_cfg_lifecycle|text|Describes the state of the file system's S3 durable data repository, if it is configured with an S3 repository|
1818
|deployment_type|text|The deployment type of the FSx for Lustre file system|
1919
|drive_cache_type|text|The type of drive cache used by PERSISTENT_1 file systems that are provisioned with HDD storage devices|
20-
|log_configuration_level|text|The data repository events that are logged by Amazon FSx. * WARN_ONLY - only warning events are logged. * ERROR_ONLY - only error events are logged. * WARN_ERROR - both warning events and error events are logged. * DISABLED - logging of data repository events is turned off. This member is required.|
20+
|log_configuration_level|text|The data repository events that are logged by Amazon FSx|
2121
|log_configuration_destination|text|The Amazon Resource Name (ARN) that specifies the destination of the logs|
2222
|mount_name|text|You use the MountName value when mounting the file system|
2323
|per_unit_storage_throughput|bigint|Per unit storage throughput represents the megabytes per second of read or write throughput per 1 tebibyte of storage provisioned|

‎plugins/source/aws/docs/tables/aws_fsx_filesystem_ontap_configuration.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
11

22
# Table: aws_fsx_filesystem_ontap_configuration
3-
Configuration for the FSx for NetApp ONTAP file system.
3+
Configuration for the FSx for NetApp ONTAP file system
44
## Columns
55
| Name | Type | Description |
66
| ------------- | ------------- | ----- |
77
|filesystem_cq_id|uuid|Unique CloudQuery ID of aws_fsx_filesystems table (FK)|
88
|automatic_backup_retention_days|bigint|The number of days to retain automatic backups|
99
|daily_automatic_backup_start_time|text|A recurring daily time, in the format HH:MM|
10-
|deployment_type|text|Specifies the FSx for ONTAP file system deployment type in use in the file system. * MULTI_AZ_1 - (Default) A high availability file system configured for Multi-AZ redundancy to tolerate temporary Availability Zone (AZ) unavailability. * SINGLE_AZ_1 - A file system configured for Single-AZ redundancy. For information about the use cases for Multi-AZ and Single-AZ deployments, refer to Choosing Multi-AZ or Single-AZ file system deployment (https://docs.aws.amazon.com/fsx/latest/ONTAPGuide/high-availability-multiAZ.html).|
11-
|disk_iops_configuration_iops|bigint|The total number of SSD IOPS provisioned for the file system.|
12-
|disk_iops_configuration_mode|text|Specifies whether the number of IOPS for the file system is using the system default (AUTOMATIC) or was provisioned by the customer (USER_PROVISIONED).|
10+
|deployment_type|text|Specifies the FSx for ONTAP file system deployment type in use in the file system|
11+
|disk_iops_configuration_iops|bigint|The total number of SSD IOPS provisioned for the file system|
12+
|disk_iops_configuration_mode|text|Specifies whether the number of IOPS for the file system is using the system default (AUTOMATIC) or was provisioned by the customer (USER_PROVISIONED)|
1313
|endpoint_ip_address_range|text|(Multi-AZ only) The IP address range in which the endpoints to access your file system are created|
1414
|endpoints_intercluster_dns_name|text|The Domain Name Service (DNS) name for the file system|
15-
|endpoints_intercluster_ip_addresses|text[]|IP addresses of the file system endpoint.|
15+
|endpoints_intercluster_ip_addresses|text[]|IP addresses of the file system endpoint|
1616
|endpoints_management_dns_name|text|The Domain Name Service (DNS) name for the file system|
17-
|endpoints_management_ip_addresses|text[]|IP addresses of the file system endpoint.|
17+
|endpoints_management_ip_addresses|text[]|IP addresses of the file system endpoint|
1818
|preferred_subnet_id|text|The ID for a subnet|
19-
|route_table_ids|text[]|(Multi-AZ only) The VPC route tables in which your file system's endpoints are created.|
20-
|throughput_capacity|bigint|The sustained throughput of an Amazon FSx file system in Megabytes per second (MBps).|
19+
|route_table_ids|text[]|(Multi-AZ only) The VPC route tables in which your file system's endpoints are created|
20+
|throughput_capacity|bigint|The sustained throughput of an Amazon FSx file system in Megabytes per second (MBps)|
2121
|weekly_maintenance_start_time|text|A recurring weekly time, in the format D:HH:MM|

‎plugins/source/aws/docs/tables/aws_fsx_filesystem_open_zfs_configuration.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
# Table: aws_fsx_filesystem_open_zfs_configuration
3-
The configuration for the Amazon FSx for OpenZFS file system.
3+
The configuration for the Amazon FSx for OpenZFS file system
44
## Columns
55
| Name | Type | Description |
66
| ------------- | ------------- | ----- |
@@ -10,8 +10,8 @@ The configuration for the Amazon FSx for OpenZFS file system.
1010
|copy_tags_to_volumes|boolean|A Boolean value indicating whether tags for the volume should be copied to snapshots|
1111
|daily_automatic_backup_start_time|text|A recurring daily time, in the format HH:MM|
1212
|deployment_type|text|Specifies the file-system deployment type|
13-
|disk_iops_configuration_iops|bigint|The total number of SSD IOPS provisioned for the file system.|
14-
|disk_iops_configuration_mode|text|Specifies whether the number of IOPS for the file system is using the system default (AUTOMATIC) or was provisioned by the customer (USER_PROVISIONED).|
15-
|root_volume_id|text|The ID of the root volume of the OpenZFS file system.|
13+
|disk_iops_configuration_iops|bigint|The total number of SSD IOPS provisioned for the file system|
14+
|disk_iops_configuration_mode|text|Specifies whether the number of IOPS for the file system is using the system default (AUTOMATIC) or was provisioned by the customer (USER_PROVISIONED)|
15+
|root_volume_id|text|The ID of the root volume of the OpenZFS file system|
1616
|throughput_capacity|bigint|The throughput of an Amazon FSx file system, measured in megabytes per second (MBps)|
1717
|weekly_maintenance_start_time|text|A recurring weekly time, in the format D:HH:MM|

‎plugins/source/aws/docs/tables/aws_fsx_filesystem_windows_configuration.md‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
11

22
# Table: aws_fsx_filesystem_windows_configuration
3-
The configuration for this Microsoft Windows file system.
3+
The configuration for this Microsoft Windows file system
44
## Columns
55
| Name | Type | Description |
66
| ------------- | ------------- | ----- |
77
|filesystem_cq_id|uuid|Unique CloudQuery ID of aws_fsx_filesystems table (FK)|
8-
|active_directory_id|text|The ID for an existing Amazon Web Services Managed Microsoft Active Directory instance that the file system is joined to.|
8+
|active_directory_id|text|The ID for an existing Amazon Web Services Managed Microsoft Active Directory instance that the file system is joined to|
99
|aliases|jsonb|An array of one or more DNS aliases that are currently associated with the Amazon FSx file system|
10-
|file_access_audit_log_level|text|Sets which attempt type is logged by Amazon FSx for file and folder accesses. * SUCCESS_ONLY - only successful attempts to access files or folders are logged. * FAILURE_ONLY - only failed attempts to access files or folders are logged. * SUCCESS_AND_FAILURE - both successful attempts and failed attempts to access files or folders are logged. * DISABLED - access auditing of files and folders is turned off. This member is required.|
11-
|file_share_access_audit_log_level|text|Sets which attempt type is logged by Amazon FSx for file share accesses. * SUCCESS_ONLY - only successful attempts to access file shares are logged. * FAILURE_ONLY - only failed attempts to access file shares are logged. * SUCCESS_AND_FAILURE - both successful attempts and failed attempts to access file shares are logged. * DISABLED - access auditing of file shares is turned off. This member is required.|
10+
|file_access_audit_log_level|text|Sets which attempt type is logged by Amazon FSx for file and folder accesses|
11+
|file_share_access_audit_log_level|text|Sets which attempt type is logged by Amazon FSx for file share accesses|
1212
|audit_log_destination|text|The Amazon Resource Name (ARN) for the destination of the audit logs|
1313
|automatic_backup_retention_days|bigint|The number of days to retain automatic backups|
1414
|copy_tags_to_backups|boolean|A boolean flag indicating whether tags on the file system should be copied to backups|
15-
|daily_automatic_backup_start_time|text|The preferred time to take daily automatic backups, in the UTC time zone.|
16-
|deployment_type|text|Specifies the file system deployment type, valid values are the following: * MULTI_AZ_1 - Specifies a high availability file system that is configured for Multi-AZ redundancy to tolerate temporary Availability Zone (AZ) unavailability, and supports SSD and HDD storage. * SINGLE_AZ_1 - (Default) Specifies a file system that is configured for single AZ redundancy, only supports SSD storage. * SINGLE_AZ_2 - Latest generation Single AZ file system|
17-
|maintenance_operations_in_progress|text[]|The list of maintenance operations in progress for this file system.|
15+
|daily_automatic_backup_start_time|text|The preferred time to take daily automatic backups, in the UTC time zone|
16+
|deployment_type|text|Specifies the file system deployment type, valid values are the following: * MULTI_AZ_1 - Specifies a high availability file system that is configured for Multi-AZ redundancy to tolerate temporary Availability Zone (AZ) unavailability, and supports SSD and HDD storage|
17+
|maintenance_operations_in_progress|text[]|The list of maintenance operations in progress for this file system|
1818
|preferred_file_server_ip|text|For MULTI_AZ_1 deployment types, the IP address of the primary, or preferred, file server|
1919
|preferred_subnet_id|text|For MULTI_AZ_1 deployment types, it specifies the ID of the subnet where the preferred file server is located|
2020
|remote_administration_endpoint|text|For MULTI_AZ_1 deployment types, use this endpoint when performing administrative tasks on the file system using Amazon FSx Remote PowerShell|
21-
|self_managed_ad_config_dns_ips|text[]|A list of up to three IP addresses of DNS servers or domain controllers in the self-managed AD directory.|
22-
|self_managed_ad_config_domain_name|text|The fully qualified domain name of the self-managed AD directory.|
23-
|self_managed_ad_config_file_system_administrators_group|text|The name of the domain group whose members have administrative privileges for the FSx file system.|
24-
|self_managed_ad_config_organizational_unit_distinguished_name|text|The fully qualified distinguished name of the organizational unit within the self-managed AD directory to which the Windows File Server or ONTAP storage virtual machine (SVM) instance is joined.|
25-
|self_managed_ad_config_user_name|text|The user name for the service account on your self-managed AD domain that FSx uses to join to your AD domain.|
26-
|throughput_capacity|bigint|The throughput of the Amazon FSx file system, measured in megabytes per second.|
21+
|self_managed_ad_config_dns_ips|text[]|A list of up to three IP addresses of DNS servers or domain controllers in the self-managed AD directory|
22+
|self_managed_ad_config_domain_name|text|The fully qualified domain name of the self-managed AD directory|
23+
|self_managed_ad_config_file_system_administrators_group|text|The name of the domain group whose members have administrative privileges for the FSx file system|
24+
|self_managed_ad_config_organizational_unit_distinguished_name|text|The fully qualified distinguished name of the organizational unit within the self-managed AD directory to which the Windows File Server or ONTAP storage virtual machine (SVM) instance is joined|
25+
|self_managed_ad_config_user_name|text|The user name for the service account on your self-managed AD domain that FSx uses to join to your AD domain|
26+
|throughput_capacity|bigint|The throughput of the Amazon FSx file system, measured in megabytes per second|
2727
|weekly_maintenance_start_time|text|The preferred start time to perform weekly maintenance, formatted d:HH:MM in the UTC time zone|

0 commit comments

Comments
 (0)