Skip to content

Commit 4933680

Browse files
author
Carlos Pérez-Aradros Herce
authored
Use CRI paths in kubernetes manifests (#12632)
We added a new `container` input in #12162, this change makes use of it to read logs from their CRI paths. Making Filebeat work with deployments that are not using Docker runtime.
1 parent fec6b95 commit 4933680

4 files changed

Lines changed: 55 additions & 68 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
211211
- Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253]
212212
- Add MSSQL module {pull}12079[12079]
213213
- Add ISO8601 date parsing support for system module. {pull}12568[12568] {pull}12578[12579]
214+
- Update Kubernetes deployment manifest to use `container` input. {pull}12632[12632]
214215
- Use correct OS path separator in `add_kubernetes_metadata` to support Windows nodes. {pull}9205[9205]
215216
- Add support for client addresses with port in Apache error logs {pull}12695[12695]
216217

‎deploy/kubernetes/filebeat-kubernetes.yaml‎

Lines changed: 27 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,28 @@ metadata:
88
k8s-app: filebeat
99
data:
1010
filebeat.yml: |-
11-
filebeat.config:
12-
inputs:
13-
# Mounted `filebeat-inputs` configmap:
14-
path: ${path.config}/inputs.d/*.yml
15-
# Reload inputs configs as they change:
16-
reload.enabled: false
17-
modules:
18-
path: ${path.config}/modules.d/*.yml
19-
# Reload module configs as they change:
20-
reload.enabled: false
11+
filebeat.inputs:
12+
- type: container
13+
paths:
14+
- /var/log/containers/*.log
15+
processors:
16+
- add_kubernetes_metadata:
17+
in_cluster: true
18+
host: ${NODE_NAME}
19+
matchers:
20+
- logs_path:
21+
logs_path: "/var/log/containers/"
2122
22-
# To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
23+
# To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
2324
#filebeat.autodiscover:
2425
# providers:
2526
# - type: kubernetes
27+
# host: ${NODE_NAME}
2628
# hints.enabled: true
29+
# hints.default_config:
30+
# type: container
31+
# paths:
32+
# - /var/log/containers/*${data.kubernetes.container.id}.log
2733
2834
processors:
2935
- add_cloud_metadata:
@@ -36,22 +42,6 @@ data:
3642
username: ${ELASTICSEARCH_USERNAME}
3743
password: ${ELASTICSEARCH_PASSWORD}
3844
---
39-
apiVersion: v1
40-
kind: ConfigMap
41-
metadata:
42-
name: filebeat-inputs
43-
namespace: kube-system
44-
labels:
45-
k8s-app: filebeat
46-
data:
47-
kubernetes.yml: |-
48-
- type: docker
49-
containers.ids:
50-
- "*"
51-
processors:
52-
- add_kubernetes_metadata:
53-
in_cluster: true
54-
---
5545
apiVersion: extensions/v1beta1
5646
kind: DaemonSet
5747
metadata:
@@ -87,6 +77,10 @@ spec:
8777
value:
8878
- name: ELASTIC_CLOUD_AUTH
8979
value:
80+
- name: NODE_NAME
81+
valueFrom:
82+
fieldRef:
83+
fieldPath: spec.nodeName
9084
securityContext:
9185
runAsUser: 0
9286
# If using Red Hat OpenShift uncomment this:
@@ -102,14 +96,14 @@ spec:
10296
mountPath: /etc/filebeat.yml
10397
readOnly: true
10498
subPath: filebeat.yml
105-
- name: inputs
106-
mountPath: /usr/share/filebeat/inputs.d
107-
readOnly: true
10899
- name: data
109100
mountPath: /usr/share/filebeat/data
110101
- name: varlibdockercontainers
111102
mountPath: /var/lib/docker/containers
112103
readOnly: true
104+
- name: varlog
105+
mountPath: /var/log
106+
readOnly: true
113107
volumes:
114108
- name: config
115109
configMap:
@@ -118,10 +112,9 @@ spec:
118112
- name: varlibdockercontainers
119113
hostPath:
120114
path: /var/lib/docker/containers
121-
- name: inputs
122-
configMap:
123-
defaultMode: 0600
124-
name: filebeat-inputs
115+
- name: varlog
116+
hostPath:
117+
path: /var/log
125118
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
126119
- name: data
127120
hostPath:

‎deploy/kubernetes/filebeat/filebeat-configmap.yaml‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,28 @@ metadata:
88
k8s-app: filebeat
99
data:
1010
filebeat.yml: |-
11-
filebeat.config:
12-
inputs:
13-
# Mounted `filebeat-inputs` configmap:
14-
path: ${path.config}/inputs.d/*.yml
15-
# Reload inputs configs as they change:
16-
reload.enabled: false
17-
modules:
18-
path: ${path.config}/modules.d/*.yml
19-
# Reload module configs as they change:
20-
reload.enabled: false
11+
filebeat.inputs:
12+
- type: container
13+
paths:
14+
- /var/log/containers/*.log
15+
processors:
16+
- add_kubernetes_metadata:
17+
in_cluster: true
18+
host: ${NODE_NAME}
19+
matchers:
20+
- logs_path:
21+
logs_path: "/var/log/containers/"
2122
22-
# To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
23+
# To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
2324
#filebeat.autodiscover:
2425
# providers:
2526
# - type: kubernetes
27+
# host: ${NODE_NAME}
2628
# hints.enabled: true
29+
# hints.default_config:
30+
# type: container
31+
# paths:
32+
# - /var/log/containers/*${data.kubernetes.container.id}.log
2733
2834
processors:
2935
- add_cloud_metadata:
@@ -35,19 +41,3 @@ data:
3541
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
3642
username: ${ELASTICSEARCH_USERNAME}
3743
password: ${ELASTICSEARCH_PASSWORD}
38-
---
39-
apiVersion: v1
40-
kind: ConfigMap
41-
metadata:
42-
name: filebeat-inputs
43-
namespace: kube-system
44-
labels:
45-
k8s-app: filebeat
46-
data:
47-
kubernetes.yml: |-
48-
- type: docker
49-
containers.ids:
50-
- "*"
51-
processors:
52-
- add_kubernetes_metadata:
53-
in_cluster: true

‎deploy/kubernetes/filebeat/filebeat-daemonset.yaml‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ spec:
3333
value:
3434
- name: ELASTIC_CLOUD_AUTH
3535
value:
36+
- name: NODE_NAME
37+
valueFrom:
38+
fieldRef:
39+
fieldPath: spec.nodeName
3640
securityContext:
3741
runAsUser: 0
3842
# If using Red Hat OpenShift uncomment this:
@@ -48,14 +52,14 @@ spec:
4852
mountPath: /etc/filebeat.yml
4953
readOnly: true
5054
subPath: filebeat.yml
51-
- name: inputs
52-
mountPath: /usr/share/filebeat/inputs.d
53-
readOnly: true
5455
- name: data
5556
mountPath: /usr/share/filebeat/data
5657
- name: varlibdockercontainers
5758
mountPath: /var/lib/docker/containers
5859
readOnly: true
60+
- name: varlog
61+
mountPath: /var/log
62+
readOnly: true
5963
volumes:
6064
- name: config
6165
configMap:
@@ -64,10 +68,9 @@ spec:
6468
- name: varlibdockercontainers
6569
hostPath:
6670
path: /var/lib/docker/containers
67-
- name: inputs
68-
configMap:
69-
defaultMode: 0600
70-
name: filebeat-inputs
71+
- name: varlog
72+
hostPath:
73+
path: /var/log
7174
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
7275
- name: data
7376
hostPath:

0 commit comments

Comments
 (0)