Skip to content

Commit 399e8b0

Browse files
leehinmanleehinman
authored
Add third-party REST API (#545)
* Add third-party REST API - apache - nginx - zeek - aws/cloudtrail
1 parent abad2be commit 399e8b0

File tree

91 files changed

+6202
-11
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+6202
-11
lines changed

‎packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
config_version: "2"
2+
interval: {{interval}}
3+
auth.basic.user: {{username}}
4+
auth.basic.password: {{password}}
5+
cursor:
6+
index_earliest:
7+
value: '[[.last_event.result.max_indextime]]'
8+
request.url: {{url}}/services/search/jobs/export
9+
{{#if ssl}}
10+
request.ssl: {{ssl}}
11+
{{/if}}
12+
request.method: POST
13+
request.transforms:
14+
- set:
15+
target: url.params.search
16+
value: |-
17+
{{search}} | streamstats max(_indextime) AS max_indextime
18+
- set:
19+
target: url.params.output_mode
20+
value: "json"
21+
- set:
22+
target: url.params.index_earliest
23+
value: '[[ .cursor.index_earliest ]]'
24+
default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
25+
- set:
26+
target: url.params.index_latest
27+
value: '[[(now).Unix]]'
28+
- set:
29+
target: header.Content-Type
30+
value: application/x-www-form-urlencoded
31+
response.decode_as: application/x-ndjson
32+
response.split:
33+
target: body.result._raw
34+
type: string
35+
delimiter: "\n"
36+
tags:
37+
{{#each tags as |tag i|}}
38+
- {{tag}}
39+
{{/each}}
40+
{{#contains tags "forwarded"}}
41+
publisher_pipeline.disable_host: true
42+
{{/contains}}
43+
processors:
44+
- decode_json_fields:
45+
fields: message
46+
target: json
47+
add_error_key: true
48+
- drop_event:
49+
when:
50+
not:
51+
has_fields: ['json.result']
52+
- fingerprint:
53+
fields:
54+
- json.result._cd
55+
- json.result._indextime
56+
- json.result._raw
57+
- json.result._time
58+
- json.result.host
59+
- json.result.source
60+
target_field: "@metadata._id"
61+
- drop_fields:
62+
fields: message
63+
- rename:
64+
fields:
65+
- from: json.result._raw
66+
to: message
67+
- from: json.result.host
68+
to: host.name
69+
- from: json.result.source
70+
to: file.path
71+
ignore_missing: true
72+
fail_on_error: false
73+
- drop_fields:
74+
fields: json
75+
- add_fields:
76+
target: ''
77+
fields:
78+
ecs.version: 1.7.0

‎packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ processors:
4040
ignore_missing: true
4141
patterns:
4242
- ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
43+
- remove:
44+
field: event.created
45+
ignore_missing: true
46+
ignore_failure: true
4347
- rename:
4448
field: '@timestamp'
4549
target_field: event.created

‎packages/apache/data_stream/access/manifest.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,29 @@ streams:
1717
template_path: log.yml.hbs
1818
title: Apache access logs
1919
description: Collect Apache access logs
20+
- input: httpjson
21+
title: Apache access logs via Splunk Enterprise REST API
22+
description: Collect apache access logs via Splunk Enterprise REST API
23+
enabled: false
24+
template_path: httpjson.yml.hbs
25+
vars:
26+
- name: interval
27+
type: text
28+
title: Interval to query Splunk Enterprise REST API
29+
description: Go Duration syntax (eg. 10s)
30+
show_user: true
31+
required: true
32+
default: 10s
33+
- name: search
34+
type: text
35+
title: Splunk search string
36+
show_user: true
37+
required: true
38+
default: "search sourcetype=\"access*\""
39+
- name: tags
40+
type: text
41+
title: Tags
42+
multi: true
43+
show_user: false
44+
default:
45+
- forwarded

‎packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
config_version: 2
2+
interval: {{interval}}
3+
auth.basic.user: {{username}}
4+
auth.basic.password: {{password}}
5+
cursor:
6+
index_earliest:
7+
value: '[[.last_event.result.max_indextime]]'
8+
request.url: {{url}}/services/search/jobs/export
9+
{{#if ssl}}
10+
request.ssl: {{ssl}}
11+
{{/if}}
12+
request.method: POST
13+
request.transforms:
14+
- set:
15+
target: url.params.search
16+
value: |-
17+
{{search}} | streamstats max(_indextime) AS max_indextime
18+
- set:
19+
target: url.params.output_mode
20+
value: "json"
21+
- set:
22+
target: url.params.index_earliest
23+
value: '[[ .cursor.index_earliest ]]'
24+
default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
25+
- set:
26+
target: url.params.index_latest
27+
value: '[[(now).Unix]]'
28+
- set:
29+
target: header.Content-Type
30+
value: application/x-www-form-urlencoded
31+
response.decode_as: application/x-ndjson
32+
response.split:
33+
target: body.result._raw
34+
type: string
35+
delimiter: "\n"
36+
tags:
37+
{{#each tags as |tag i|}}
38+
- {{tag}}
39+
{{/each}}
40+
{{#contains tags "forwarded"}}
41+
publisher_pipeline.disable_host: true
42+
{{/contains}}
43+
processors:
44+
- decode_json_fields:
45+
fields: message
46+
target: json
47+
add_error_key: true
48+
- drop_event:
49+
when:
50+
not:
51+
has_fields: ['json.result']
52+
- fingerprint:
53+
fields:
54+
- json.result._cd
55+
- json.result._indextime
56+
- json.result._raw
57+
- json.result._time
58+
- json.result.host
59+
- json.result.source
60+
target_field: "@metadata._id"
61+
- drop_fields:
62+
fields: message
63+
- rename:
64+
fields:
65+
- from: json.result._raw
66+
to: message
67+
- from: json.result.host
68+
to: host.name
69+
- from: json.result.source
70+
to: file.path
71+
ignore_missing: true
72+
fail_on_error: false
73+
- drop_fields:
74+
fields: json
75+
- add_fields:
76+
target: ''
77+
fields:
78+
ecs.version: 1.7.0

‎packages/apache/data_stream/error/manifest.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,29 @@ streams:
1616
template_path: log.yml.hbs
1717
title: Apache error logs
1818
description: Collect Apache error logs
19+
- input: httpjson
20+
title: Apache error logs via Splunk Enterprise REST API
21+
description: Collect apache error logs via Splunk Enterprise REST API
22+
enabled: false
23+
template_path: httpjson.yml.hbs
24+
vars:
25+
- name: interval
26+
type: text
27+
title: Interval to query Splunk Enterprise REST API
28+
description: Go Duration syntax (eg. 10s)
29+
show_user: true
30+
required: true
31+
default: 10s
32+
- name: search
33+
type: text
34+
title: Splunk search string
35+
show_user: true
36+
required: true
37+
default: search sourcetype=apache:error OR sourcetype=apache_error
38+
- name: tags
39+
type: text
40+
title: Tags
41+
multi: true
42+
show_user: false
43+
default:
44+
- forwarded

‎packages/apache/manifest.yml‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
format_version: 1.0.0
22
name: apache
33
title: Apache
4-
version: 0.3.4
4+
version: 0.4.0
55
license: basic
66
description: Apache Integration
77
type: integration
88
categories:
99
- web
1010
release: experimental
1111
conditions:
12-
kibana.version: '^7.11.0'
12+
kibana.version: '^7.12.0'
1313
screenshots:
1414
- src: /img/apache-metrics-overview.png
1515
title: Apache metrics overview
@@ -32,6 +32,34 @@ policy_templates:
3232
- type: logfile
3333
title: Collect logs from Apache instances
3434
description: Collecting Apache access and error logs
35+
- type: httpjson
36+
title: Collect logs from third-party REST API (experimental)
37+
description: Collect logs from third-party REST API (experimental)
38+
vars:
39+
- name: url
40+
type: text
41+
title: URL of Splunk Enterprise Server
42+
description: i.e. scheme://host:port, path is automatic
43+
show_user: true
44+
required: true
45+
default: https://server.example.com:8089
46+
- name: username
47+
type: text
48+
title: Splunk REST API Username
49+
show_user: true
50+
required: true
51+
- name: password
52+
type: password
53+
title: Splunk REST API Password
54+
required: true
55+
show_user: true
56+
- name: ssl
57+
type: yaml
58+
title: SSL Configuration
59+
description: i.e. certificate_authorities, supported_protocols, verification_mode etc.
60+
multi: false
61+
required: false
62+
show_user: false
3563
- type: apache/metrics
3664
title: Collect metrics from Apache instances
3765
description: Collecting Apache status metrics

‎packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
config_version: 2
2+
interval: {{interval}}
3+
auth.basic.user: {{username}}
4+
auth.basic.password: {{password}}
5+
cursor:
6+
index_earliest:
7+
value: '[[.last_event.result.max_indextime]]'
8+
request.url: {{url}}/services/search/jobs/export
9+
{{#if ssl}}
10+
request.ssl: {{ssl}}
11+
{{/if}}
12+
request.method: POST
13+
request.transforms:
14+
- set:
15+
target: url.params.search
16+
value: {{search}} | streamstats max(_indextime) AS max_indextime
17+
- set:
18+
target: url.params.output_mode
19+
value: "json"
20+
- set:
21+
target: url.params.index_earliest
22+
value: '[[ .cursor.index_earliest ]]'
23+
default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
24+
- set:
25+
target: url.params.index_latest
26+
value: '[[(now).Unix]]'
27+
- set:
28+
target: header.Content-Type
29+
value: application/x-www-form-urlencoded
30+
response.decode_as: application/x-ndjson
31+
response.split:
32+
target: body.result._raw
33+
type: string
34+
delimiter: "\n"
35+
{{#contains tags "forwarded"}}
36+
publisher_pipeline.disable_host: true
37+
{{/contains}}
38+
tags:
39+
{{#each tags as |tag i|}}
40+
- {{tag}}
41+
{{/each}}
42+
processors:
43+
- decode_json_fields:
44+
fields: message
45+
target: json
46+
add_error_key: true
47+
- drop_event:
48+
when:
49+
not:
50+
has_fields: ['json.result']
51+
- fingerprint:
52+
fields:
53+
- json.result._cd
54+
- json.result._indextime
55+
- json.result._raw
56+
- json.result._time
57+
target_field: "@metadata._id"
58+
- drop_fields:
59+
fields: ["message"]
60+
- rename:
61+
fields:
62+
- from: json.result._raw
63+
to: message
64+
- drop_fields:
65+
fields: ["json"]
66+
ignore_missing: true
67+
- add_fields:
68+
target: ''
69+
fields:
70+
ecs.version: 1.8.0

‎packages/aws/data_stream/cloudtrail/manifest.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,29 @@ streams:
2222
required: false
2323
show_user: false
2424
description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint.
25+
- input: httpjson
26+
title: AWS CloudTrail logs via Splunk Enterprise REST API
27+
description: Collect AWS CloudTrail logs via Splunk Enterprise REST API
28+
enabled: false
29+
template_path: httpjson.yml.hbs
30+
vars:
31+
- name: interval
32+
type: text
33+
title: Interval to query Splunk Enterprise REST API
34+
description: Go Duration syntax (eg. 10s)
35+
show_user: true
36+
required: true
37+
default: 10s
38+
- name: search
39+
type: text
40+
title: Splunk search string
41+
show_user: true
42+
required: true
43+
default: "search sourcetype=aws:cloudtrail"
44+
- name: tags
45+
type: text
46+
title: Tags
47+
multi: true
48+
show_user: false
49+
default:
50+
- forwarded

0 commit comments

Comments
 (0)