Skip to content

Commit 751926d

Browse files
Davide GirardiDavide Girardi
authored andcommitted
Added documentation and remove field processor
1 parent 13b9b55 commit 751926d

File tree

5 files changed

+430
-130
lines changed

5 files changed

+430
-130
lines changed
Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1-
# Redshift metrics
1+
# Amazon Redshift
22

3-
Test
3+
This integration is used to fetch metrics from [Amazon Redshift](https://aws.amazon.com/redshift/).
4+
5+
## Metrics
6+
7+
The `redshift` dataset collects Amazon Redshift metrics.
8+
9+
{{event "redshift" }}
10+
11+
{{fields "redshift"}}

packages/aws/data_stream/redshift/elasticsearch/ingest_pipeline/default.yml

Lines changed: 43 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,125 +2,138 @@
22
description: "Ingest Pipeline for Amazon Redshift metrics"
33

44
processors:
5+
- rename:
6+
field: aws.dimensions.service class
7+
target_field: aws.dimensions.service_class
8+
ignore_missing: true
59
- rename:
610
field: aws.redshift.metrics.CommitQueueLength.avg
7-
target_field: aws.redshift.metrics.commit_queue_length
11+
target_field: aws.redshift.status.commit_queue_length
812
ignore_missing: true
913
- rename:
1014
field: aws.redshift.metrics.ConcurrencyScalingActiveClusters.avg
11-
target_field: aws.redshift.metrics.concurrency_scaling_active_clusters
15+
target_field: aws.redshift.performance.concurrency_scaling_active_clusters
1216
ignore_missing: true
1317
- rename:
1418
field: aws.redshift.metrics.ConcurrencyScalingSeconds.avg
15-
target_field: aws.redshift.metrics.concurrency_scaling_seconds
19+
target_field: aws.redshift.performance.concurrency_scaling_seconds
1620
ignore_missing: true
1721
- rename:
1822
field: aws.redshift.metrics.CPUUtilization.avg
19-
target_field: aws.redshift.metrics.cpu_utilization
23+
target_field: aws.redshift.cpu.cpu_utilization
2024
ignore_missing: true
2125
- rename:
2226
field: aws.redshift.metrics.DatabaseConnections.avg
23-
target_field: aws.redshift.metrics.database_connections
27+
target_field: aws.redshift.status.database_connections
2428
ignore_missing: true
2529
- rename:
2630
field: aws.redshift.metrics.HealthStatus.avg
27-
target_field: aws.redshift.metrics.health_status
31+
target_field: aws.redshift.status.health_status
2832
ignore_missing: true
2933
- rename:
3034
field: aws.redshift.metrics.MaintenanceMode.avg
31-
target_field: aws.redshift.metrics.maintenance_mode
35+
target_field: aws.redshift.status.maintenance_mode
3236
ignore_missing: true
3337
- rename:
3438
field: aws.redshift.metrics.MaxConfiguredConcurrencyScalingClusters.avg
35-
target_field: aws.redshift.metrics.max_configured_concurrency_scaling_clusters
39+
target_field: aws.redshift.performance.max_configured_concurrency_scaling_clusters
3640
ignore_missing: true
3741
- rename:
3842
field: aws.redshift.metrics.NetworkReceiveThroughput.avg
39-
target_field: aws.redshift.metrics.network_receive_throughput
43+
target_field: aws.redshift.network.network_receive_throughput
4044
ignore_missing: true
4145
- rename:
4246
field: aws.redshift.metrics.NetworkTransmitThroughput.avg
43-
target_field: aws.redshift.metrics.network_transmit_throughput
47+
target_field: aws.redshift.network.network_transmit_throughput
4448
ignore_missing: true
4549
- rename:
4650
field: aws.redshift.metrics.PercentageDiskSpaceUsed.avg
47-
target_field: aws.redshift.metrics.percentage_disk_space_used
51+
target_field: aws.redshift.storage.percentage_disk_space_used
4852
ignore_missing: true
4953
- rename:
5054
field: aws.redshift.metrics.QueriesCompletedPerSecond.avg
51-
target_field: aws.redshift.metrics.queries_completed_per_second
55+
target_field: aws.redshift.performance.queries_completed_per_second
5256
ignore_missing: true
5357
- rename:
5458
field: aws.redshift.metrics.QueryDuration.avg
55-
target_field: aws.redshift.metrics.query_duration
59+
target_field: aws.redshift.performance.query_duration
5660
ignore_missing: true
5761
- rename:
5862
field: aws.redshift.metrics.QueryRuntimeBreakdown.avg
59-
target_field: aws.redshift.metrics.query_runtime_breakdown
63+
target_field: aws.redshift.performance.query_runtime_breakdown
6064
ignore_missing: true
6165
- rename:
6266
field: aws.redshift.metrics.ReadIOPS.avg
63-
target_field: aws.redshift.metrics.read_iops
67+
target_field: aws.redshift.storage.read_iops
6468
ignore_missing: true
6569
- rename:
6670
field: aws.redshift.metrics.ReadLatency.avg
67-
target_field: aws.redshift.metrics.read_latency
71+
target_field: aws.redshift.storage.read_latency
6872
ignore_missing: true
6973
- rename:
7074
field: aws.redshift.metrics.ReadThroughput.avg
71-
target_field: aws.redshift.metrics.read_throughput
75+
target_field: aws.redshift.storage.read_throughput
7276
ignore_missing: true
7377
- rename:
7478
field: aws.redshift.metrics.TotalTableCount.avg
75-
target_field: aws.redshift.metrics.total_table_count
79+
target_field: aws.redshift.status.total_table_count
7680
ignore_missing: true
7781
- rename:
7882
field: aws.redshift.metrics.WLMQueueLength.avg
79-
target_field: aws.redshift.metrics.wlm_queue_length
83+
target_field: aws.redshift.performance.wlm_queue_length
8084
ignore_missing: true
8185
- rename:
8286
field: aws.redshift.metrics.WLMQueueWaitTime.avg
83-
target_field: aws.redshift.metrics.wlm_queue_wait_time
87+
target_field: aws.redshift.performance.wlm_queue_wait_time
8488
ignore_missing: true
8589
- rename:
8690
field: aws.redshift.metrics.WLMQueriesCompletedPerSecond.avg
87-
target_field: aws.redshift.metrics.wlm_queries_completed_per_second
91+
target_field: aws.redshift.performance.wlm_queries_completed_per_second
8892
ignore_missing: true
8993
- rename:
9094
field: aws.redshift.metrics.WLMQueryDuration.avg
91-
target_field: aws.redshift.metrics.wlm_query_duration
95+
target_field: aws.redshift.performance.wlm_query_duration
9296
ignore_missing: true
9397
- rename:
9498
field: aws.redshift.metrics.WLMRunningQueries.avg
95-
target_field: aws.redshift.metrics.wlm_running_queries
99+
target_field: aws.redshift.performance.wlm_running_queries
96100
ignore_missing: true
97101
- rename:
98102
field: aws.redshift.metrics.WriteIOPS.avg
99-
target_field: aws.redshift.metrics.write_iops
103+
target_field: aws.redshift.storage.write_iops
100104
ignore_missing: true
101105
- rename:
102106
field: aws.redshift.metrics.WriteLatency.avg
103-
target_field: aws.redshift.metrics.write_latency
107+
target_field: aws.redshift.storage.write_latency
104108
ignore_missing: true
105109
- rename:
106110
field: aws.redshift.metrics.WriteThroughput.avg
107-
target_field: aws.redshift.metrics.write_throughput
111+
target_field: aws.redshift.storage.write_throughput
108112
ignore_missing: true
109113
- rename:
110114
field: aws.redshift.metrics.SchemaQuota.avg
111-
target_field: aws.redshift.metrics.schema_quota
115+
target_field: aws.redshift.storage.schema_quota
112116
ignore_missing: true
113117
- rename:
114118
field: aws.redshift.metrics.NumExceededSchemaQuotas.avg
115-
target_field: aws.redshift.metrics.num_exceeds_schema_quotas
119+
target_field: aws.redshift.storages.num_exceeds_schema_quotas
116120
ignore_missing: true
117121
- rename:
118122
field: aws.redshift.metrics.StorageUsed.avg
119-
target_field: aws.redshift.metrics.storage_used
123+
target_field: aws.redshift.storage.storage_used
120124
ignore_missing: true
121125
- rename:
122126
field: aws.redshift.metrics.PercentageQuotaUsed.avg
123-
target_field: aws.redshift.metrics.percentage_quota_used
127+
target_field: aws.redshift.storage.percentage_quota_used
128+
ignore_missing: true
129+
- rename:
130+
field: aws.redshift.metrics.AutoVacuumSpaceFreed.avg
131+
target_field: aws.redshift.storage.auto_vacuum_space_freed
132+
ignore_missing: true
133+
- remove:
134+
field:
135+
- aws.redshift.metrics
136+
- aws.dimensions.service class
124137
ignore_missing: true
125138

126139

packages/aws/data_stream/redshift/fields/fields.yml

Lines changed: 96 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
type: keyword
1616
description: This dimension filters the data that you request for a specific Cluster identifier
1717
# Commented out for now because elastic-package check fails (the regex check does not accept spaces)
18-
# - name: service class
19-
# type: keyword
20-
# description: This dimension filters the data that you request for a specific WLM service class
18+
- name: service_class
19+
type: keyword
20+
description: This dimension filters the data that you request for a specific WLM service class
2121
- name: stage
2222
type: keyword
2323
description: This dimension filters the data that you request for a specific execution stage for a query.
@@ -33,99 +33,99 @@
3333
- name: redshift
3434
type: group
3535
fields:
36-
- name: metrics
37-
type: group
38-
fields:
39-
- name: commit_queue_length
40-
type: long
41-
description: The number of transactions waiting to commit at a given point in time.
42-
- name: concurrency_scaling_active_clusters
43-
type: long
44-
description: The number of concurrency scaling clusters that are actively processing queries at any given time.
45-
- name: concurrency_scaling_seconds
46-
type: long
47-
description: The number of seconds used by concurrency scaling clusters that have active query processing activity.
48-
- name: cpu_utilization
49-
type: scaled_float
50-
description: The percentage of CPU utilization. For clusters, this metric represents an aggregation of all nodes (leader and compute) CPU utilization values.
51-
- name: database_connections
52-
type: long
53-
description: The number of database connections to a cluster.
54-
- name: health_status
55-
type: scaled_float
56-
description: Indicates the health of the cluster. Any value below 1 implies that the cluster was unhealthy
57-
- name: maintenance_mode
58-
type: scaled_float
59-
description: Indicates whether the cluster is in maintenance mode. Any value greater than 0 means that the cluster was in maintenance mode.
60-
- name: max_configured_concurrency_scaling_clusters
61-
type: long
62-
description: Maximum number of concurrency scaling clusters configured from the parameter group.
63-
- name: network_receive_throughput
64-
type: long
65-
description: The rate at which the node or cluster receives data.
66-
- name: network_transmit_throughput
67-
type: long
68-
description: The rate at which the node or cluster writes data.
69-
- name: percentage_disk_space_used
70-
type: scaled_float
71-
description: The percent of disk space used.
72-
- name: queries_completed_per_second
73-
type: long
74-
description: The average number of queries completed per second.
75-
- name: queries_duration
76-
type: long
77-
description: The average amount of time to complete a query.
78-
- name: query_runtime_breakdown
79-
type: long
80-
description: The total time queries spent running by query stage.
81-
- name: read_iops
82-
type: long
83-
description: The average number of disk read operations per second.
84-
- name: read_latency
85-
type: long
86-
description: The average amount of time taken for disk read I/O operations.
87-
- name: read_throughput
88-
type: long
89-
description: The average number of bytes read from disk per second.
90-
- name: total_table_count
91-
type: long
92-
description: The number of user tables open at a particular point in time.
93-
- name: wlm_queue_length
94-
type: long
95-
description: The number of queries waiting to enter a workload management (WLM) queue.
96-
- name: wlm_queue_wait_time
97-
type: long
98-
description: The total time queries spent waiting in the workload management (WLM) queue.
99-
- name: wlm_queries_completed_per_second
100-
type: long
101-
description: The average number of queries completed per second for a workload management (WLM) queue.
102-
- name: wlm_query_duration
103-
type: long
104-
description: The average length of time to complete a query for a workload management (WLM) queue.
105-
- name: wlm_running_queries
106-
type: long
107-
description: The number of queries running from both the main cluster and concurrency scaling cluster per WLM queue.
108-
- name: write_iops
109-
type: long
110-
description: The average number of write operations per second.
111-
- name: write_latency
112-
type: long
113-
description: The average amount of time taken for disk write I/O operations.
114-
- name: write_throughput
115-
type: long
116-
description: The average number of bytes written to disk per second.
117-
- name: schema_quota
118-
type: long
119-
description: The configured quota for a schema.
120-
- name: num_exceeds_schema_quotas
121-
type: long
122-
description: The number of schemas with exceeded quotas.
123-
- name: storage_used
124-
type: long
125-
description: The disk or storage space used by a schema.
126-
- name: percentage_quota_used
127-
type: long
128-
description: The percentage of disk or storage space used relative to the configured schema quota.
36+
- name: status.commit_queue_length
37+
type: long
38+
description: The number of transactions waiting to commit at a given point in time.
39+
- name: performance.concurrency_scaling_active_clusters
40+
type: long
41+
description: The number of concurrency scaling clusters that are actively processing queries at any given time.
42+
- name: performance.concurrency_scaling_seconds
43+
type: long
44+
description: The number of seconds used by concurrency scaling clusters that have active query processing activity.
45+
- name: cpu.cpu_utilization
46+
type: scaled_float
47+
description: The percentage of CPU utilization. For clusters, this metric represents an aggregation of all nodes (leader and compute) CPU utilization values.
48+
- name: status.database_connections
49+
type: long
50+
description: The number of database connections to a cluster.
51+
- name: status.health_status
52+
type: scaled_float
53+
description: Indicates the health of the cluster. Any value below 1 implies that the cluster was unhealthy
54+
- name: status.maintenance_mode
55+
type: scaled_float
56+
description: Indicates whether the cluster is in maintenance mode. Any value greater than 0 means that the cluster was in maintenance mode.
57+
- name: performance.max_configured_concurrency_scaling_clusters
58+
type: long
59+
description: Maximum number of concurrency scaling clusters configured from the parameter group.
60+
- name: network.network_receive_throughput
61+
type: long
62+
description: The rate at which the node or cluster receives data.
63+
- name: network.network_transmit_throughput
64+
type: long
65+
description: The rate at which the node or cluster writes data.
66+
- name: storage.percentage_disk_space_used
67+
type: scaled_float
68+
description: The percent of disk space used.
69+
- name: performance.queries_completed_per_second
70+
type: long
71+
description: The average number of queries completed per second.
72+
- name: performance.query_duration
73+
type: long
74+
description: The average amount of time to complete a query.
75+
- name: performance.query_runtime_breakdown
76+
type: long
77+
description: The total time queries spent running by query stage.
78+
- name: storage.read_iops
79+
type: long
80+
description: The average number of disk read operations per second.
81+
- name: storage.read_latency
82+
type: long
83+
description: The average amount of time taken for disk read I/O operations.
84+
- name: storage.read_throughput
85+
type: long
86+
description: The average number of bytes read from disk per second.
87+
- name: status.total_table_count
88+
type: long
89+
description: The number of user tables open at a particular point in time.
90+
- name: performance.wlm_queue_length
91+
type: long
92+
description: The number of queries waiting to enter a workload management (WLM) queue.
93+
- name: performance.wlm_queue_wait_time
94+
type: long
95+
description: The total time queries spent waiting in the workload management (WLM) queue.
96+
- name: performance.wlm_queries_completed_per_second
97+
type: long
98+
description: The average number of queries completed per second for a workload management (WLM) queue.
99+
- name: performance.wlm_query_duration
100+
type: long
101+
description: The average length of time to complete a query for a workload management (WLM) queue.
102+
- name: performance.wlm_running_queries
103+
type: long
104+
description: The number of queries running from both the main cluster and concurrency scaling cluster per WLM queue.
105+
- name: storage.write_iops
106+
type: long
107+
description: The average number of write operations per second.
108+
- name: storage.write_latency
109+
type: long
110+
description: The average amount of time taken for disk write I/O operations.
111+
- name: storage.write_throughput
112+
type: long
113+
description: The average number of bytes written to disk per second.
114+
- name: storage.schema_quota
115+
type: long
116+
description: The configured quota for a schema.
117+
- name: storages.num_exceeds_schema_quotas
118+
type: long
119+
description: The number of schemas with exceeded quotas.
120+
- name: storage.storage_used
121+
type: long
122+
description: The disk or storage space used by a schema.
123+
- name: storage.percentage_quota_used
124+
type: long
125+
description: The percentage of disk or storage space used relative to the configured schema quota.
126+
- name: storage.auto_vacuum_space_freed
127+
type: long
128+
description: Storage space reclaimed by auto vacuum delete operations.
129129
- name: aws.cloudwatch.namespace
130130
type: keyword
131131
description: The namespace specified when query cloudwatch api.

0 commit comments

Comments
 (0)