Skip to content

[Agent] Add agent standalone manifests for system module & Pod's log collection #23938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChrsMark merged 12 commits into from
Feb 22, 2021
Merged

[Agent] Add agent standalone manifests for system module & Pod's log collection #23938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
fc9e7fe
Add agent standalone manifests for system module
ChrsMark Feb 9, 2021
3aec0b3
Rename file
ChrsMark Feb 16, 2021
33df263
Merge remote-tracking branch 'upstream/master' into add_standalone_ag…
ChrsMark Feb 16, 2021
5fdd099
rebase on top of k8s module
ChrsMark Feb 16, 2021
34da3b2
Split manifests as per pattern
ChrsMark Feb 16, 2021
e414c59
Add Pod's logs collection
ChrsMark Feb 17, 2021
7573987
Update configmap subfile
ChrsMark Feb 17, 2021
97018db
Add more verobse logging
ChrsMark Feb 17, 2021
c5340e3
Fix version placeholders
ChrsMark Feb 18, 2021
75c6f16
Remove ids & set log selector
ChrsMark Feb 19, 2021
e2b008f
Remove verbose logging
ChrsMark Feb 19, 2021
292c360
Remove secrets from clusterrole
ChrsMark Feb 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 165 additions & 80 deletions deploy/kubernetes/elastic-agent-standalone-kubernetes.yml
View file
Open in desktop

Large diffs are not rendered by default.

231 changes: 231 additions & 0 deletions deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: agent-node-datastreams
namespace: kube-system
labels:
k8s-app: elastic-agent
data:
agent.yml: |-
outputs:
default:
type: elasticsearch
hosts:
- >-
${ES_HOST}
username: ${ES_USERNAME}
password: ${ES_PASSWORD}
agent:
monitoring:
enabled: true
use_output: default
logs: true
metrics: true
providers.kubernetes:
node: ${NODE_NAME}
scope: node
inputs:
- name: log-1
type: logfile
use_output: default
meta:
package:
name: log
version: 0.4.6
data_stream:
namespace: default
streams:
- data_stream:
dataset: generic
symlinks: true
paths:
- /var/log/containers/*${kubernetes.container.id}.log
- name: system-3
type: system/metrics
use_output: default
meta:
package:
name: system
version: 0.10.9
data_stream:
namespace: default
streams:
- data_stream:
dataset: system.core
type: metrics
metricsets:
- core
core.metrics:
- percentages
- data_stream:
dataset: system.cpu
type: metrics
period: 10s
cpu.metrics:
- percentages
- normalized_percentages
metricsets:
- cpu
- data_stream:
dataset: system.diskio
type: metrics
period: 10s
diskio.include_devices: null
metricsets:
- diskio
- data_stream:
dataset: system.filesystem
type: metrics
period: 1m
metricsets:
- filesystem
processors:
- drop_event.when.regexp:
system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- data_stream:
dataset: system.fsstat
type: metrics
period: 1m
metricsets:
- fsstat
processors:
- drop_event.when.regexp:
system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- data_stream:
dataset: system.load
type: metrics
period: 10s
metricsets:
- load
- data_stream:
dataset: system.memory
type: metrics
period: 10s
metricsets:
- memory
- data_stream:
dataset: system.network
type: metrics
period: 10s
network.interfaces: null
metricsets:
- network
- data_stream:
dataset: system.process
type: metrics
process.include_top_n.by_memory: 5
period: 10s
processes:
- .*
process.include_top_n.by_cpu: 5
process.cgroups.enabled: false
process.cmdline.cache.enabled: true
metricsets:
- process
process.include_cpu_ticks: false
system.hostfs: /hostfs
- data_stream:
dataset: system.process_summary
type: metrics
period: 10s
metricsets:
- process_summary
system.hostfs: /hostfs
- data_stream:
dataset: system.socket_summary
type: metrics
period: 10s
metricsets:
- socket_summary
system.hostfs: /hostfs
- name: kubernetes-node-metrics
type: kubernetes/metrics
use_output: default
meta:
package:
name: kubernetes
version: 0.2.8
data_stream:
namespace: default
streams:
- data_stream:
dataset: kubernetes.controllermanager
type: metrics
metricsets:
- controllermanager
hosts:
- '${kubernetes.pod.ip}:10252'
period: 10s
condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager'
- data_stream:
dataset: kubernetes.scheduler
type: metrics
metricsets:
- scheduler
hosts:
- '${kubernetes.pod.ip}:10251'
period: 10s
condition: ${kubernetes.pod.labels.component} == 'kube-scheduler'
- data_stream:
dataset: kubernetes.proxy
type: metrics
metricsets:
- proxy
hosts:
- 'localhost:10249'
period: 10s
- data_stream:
dataset: kubernetes.container
type: metrics
metricsets:
- container
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.node
type: metrics
metricsets:
- node
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.pod
type: metrics
metricsets:
- pod
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.system
type: metrics
metricsets:
- system
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.volume
type: metrics
metricsets:
- volume
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
70 changes: 70 additions & 0 deletions deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: elastic-agent
namespace: kube-system
labels:
app: elastic-agent
spec:
selector:
matchLabels:
app: elastic-agent
template:
metadata:
labels:
app: elastic-agent
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: elastic-agent
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: elastic-agent
image: docker.elastic.co/beats/elastic-agent:%VERSION%
args: [
"-c", "/etc/agent.yml",
"-e",
]
env:
- name: ES_USERNAME
value: "elastic"
- name: ES_PASSWORD
value: ""
- name: ES_HOST
value: ""
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: datastreams
mountPath: /etc/agent.yml
readOnly: true
subPath: agent.yml
- name: proc
mountPath: /hostfs/proc
readOnly: true
- name: cgroup
mountPath: /hostfs/sys/fs/cgroup
readOnly: true
volumes:
- name: datastreams
configMap:
defaultMode: 0640
name: agent-node-datastreams
- name: proc
hostPath:
path: /proc
- name: cgroup
hostPath:
path: /sys/fs/cgroup
Loading