{"id":2769,"date":"2023-09-18T23:40:52","date_gmt":"2023-09-18T15:40:52","guid":{"rendered":"https:\/\/199604.com\/?p=2769"},"modified":"2023-09-18T23:40:52","modified_gmt":"2023-09-18T15:40:52","slug":"kube-prometheus-%e7%9b%91%e6%8e%a7elasticsearch%e6%ad%a5%e9%aa%a4","status":"publish","type":"post","link":"https:\/\/199604.com\/2769","title":{"rendered":"kube-prometheus \u76d1\u63a7ElasticSearch\u6b65\u9aa4"},"content":{"rendered":"

kube-prometheus \u76d1\u63a7ElasticSearch\u6b65\u9aa4<\/h1>\n

\u6b64\u65b9\u6cd5\u65e2\u53ef\u4ee5\u76d1\u63a7\u96c6\u7fa4\u5185\u7684ElasticSearch,\u4e5f\u53ef\u4ee5\u76d1\u63a7\u96c6\u7fa4\u5916\u7684ElasticSearch<\/code>\uff0c\u539f\u7406\u548cnode-exporter<\/code>\u4e00\u6837<\/p>\n

\u64cd\u4f5c\u6b65\u9aa4<\/h2>\n

\u521b\u5efaes-exporter<\/strong><\/h3>\n

es-exporter.yaml<\/code><\/h4>\n
apiVersion: apps\/v1\nkind: Deployment\nmetadata:\n  name: es-exporter\n  namespace: logging\nspec:\n  replicas: 1\n  revisionHistoryLimit: 2\n  strategy:\n    rollingUpdate:\n      maxSurge: 1\n      maxUnavailable: 0\n    type: RollingUpdate\n  selector:\n    matchLabels:\n      app: es-exporter\n  template:\n    metadata:\n      labels:\n        app: es-exporter\n    spec:\n\n      imagePullSecrets:\n        - name: hub\n      containers:\n        # \u53c2\u8003 https:\/\/github.com\/prometheus-community\/elasticsearch_exporter\n        - command:\n            - \/bin\/elasticsearch_exporter\n            - --es.all\n            - --es.timeout=10s\n            - --es.uri=http:\/\/username:passwd@elk-elasticsearch:9200\n          env:\n            - name: TZ\n              value: Asia\/Shanghai\n\n          image: 10.194.24.53\/k8s-component\/prometheuscommunity\/elasticsearch-exporter:v1.6.0\n          securityContext:\n            capabilities:\n              drop:\n                - SETPCAP\n                - MKNOD\n                - AUDIT_WRITE\n                - CHOWN\n                - NET_RAW\n                - DAC_OVERRIDE\n                - FOWNER\n                - FSETID\n                - KILL\n                - SETGID\n                - SETUID\n                - NET_BIND_SERVICE\n                - SYS_CHROOT\n                - SETFCAP\n            readOnlyRootFilesystem: true\n          livenessProbe:\n            httpGet:\n              path: \/healthz\n              port: 9114\n            initialDelaySeconds: 30\n            timeoutSeconds: 10\n          name: es-exporter\n          ports:\n            - containerPort: 9114\n              name: http\n          readinessProbe:\n            httpGet:\n              path: \/healthz\n              port: 9114\n            initialDelaySeconds: 10\n            timeoutSeconds: 10\n          resources:\n            limits:\n              cpu: 100m\n              memory: 128Mi\n            requests:\n              cpu: 25m\n              memory: 64Mi\n          volumeMounts:\n            - mountPath: \/etc\/localtime\n              name: localtime\n      restartPolicy: Always\n      securityContext:\n        runAsNonRoot: true\n        runAsGroup: 10000\n        runAsUser: 10000\n        fsGroup: 10000\n      volumes:\n        - name: localtime\n          hostPath:\n            path: \/etc\/localtime\n<\/code><\/pre>\n
es-exporter-svc.yaml<\/code><\/h4>\n
apiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    app: es-exporter\n  name: es-exporter-svc\n  namespace: logging\nspec:\n  ports:\n    - name: http-metrics\n      port: 9114\n      protocol: TCP\n      targetPort: 9114\n  type: ClusterIP\n  selector:\n    app: es-exporter\n<\/code><\/pre>\n
\u521b\u5efaservicemonitor<\/code>\u7684crd\u5bf9\u8c61<\/strong><\/h3>\n
apiVersion: monitoring.coreos.com\/v1\nkind: ServiceMonitor\nmetadata:\n  labels:\n    app: es-exporter\n  name: es-exporter-sm\n  namespace: logging\nspec:\n  endpoints:\n    - bearerTokenFile: \/var\/run\/secrets\/kubernetes.io\/serviceaccount\/token\n      port: http-metrics\n      interval: 30s\n  jobLabel: es-exporter\n  namespaceSelector:\n    matchNames:\n      - logging\n  selector:\n    matchLabels:\n      app: es-exporter\n<\/code><\/pre>\n
\u67e5\u770btarget<\/strong><\/h3>\n
prometheus<\/code>\u81ea\u52a8\u53d1\u73b0\u4e86es-exporter<\/code><\/p>\n
\"image-20230918233007783\"<\/div><\/p>\n
\u767b\u5f55grafana,\u5bfc\u5165\u6a21\u677f<\/strong><\/h3>\n
\u53c2\u8003\uff1ahttps:\/\/grafana.com\/grafana\/dashboards\/2322<\/code><\/p>\n
\u89e3\u51b3Cluster health\u76d1\u63a7\u663e\u793a\u4e3aN\/A<\/strong> \u95ee\u9898<\/strong><\/h4>\n
\u9009\u4e2dcluster health<\/code>\uff0c\u9009\u62e9Edit<\/p>\n
\"image-20230918233146629\"<\/div><\/p>\n
\u9f20\u6807\u79fb\u52a8\u5230\u53f3\u6d4b\uff0c\u4fee\u6539\u4e0b\u62c9\u6846\u5185\u7684\u503c<\/p>\n
\"image-20230918233212696\"<\/div><\/p>\n
\u6539\u4e3a\u5982\u4e0b\u56fe\u6240\u793a\uff0c\u7136\u540e\u70b9\u51fb\uff0capply,save<\/code>\u4fdd\u5b58\u6a21\u677f<\/p>\n
\"image-20230918233255331\"<\/div><\/p>\n
\u914d\u7f6eprometheusrule<\/strong><\/h3>\n
\u521b\u5efaes-rule.yaml<\/code><\/p>\n
\u56e0\u4e3a\u6211es\u8282\u70b9\u5c312\u4e2a\uff0c\u6240\u4ee5\u6ce8\u91ca\u4e86\u90e8\u5206\u544a\u8b66\u89c4\u5219<\/p>\n
apiVersion: monitoring.coreos.com\/v1\nkind: PrometheusRule\nmetadata:\n  labels:\n    prometheus: k8s\n    role: alert-rules\n  name: es-rules\n  namespace: monitoring\nspec:\n  groups:\n    - name: es.rules\n      rules:\n        - alert: ElasticsearchHeapUsageTooHigh\n          expr: (elasticsearch_jvm_memory_used_bytes{area=\"heap\"} \/ elasticsearch_jvm_memory_max_bytes{area=\"heap\"}) * 100 > 90\n          for: 2m\n          labels:\n            severity: critical\n          annotations:\n            summary: Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})\n            description: \"The heap usage is over 90%\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchHeapUsageWarning\n          expr: (elasticsearch_jvm_memory_used_bytes{area=\"heap\"} \/ elasticsearch_jvm_memory_max_bytes{area=\"heap\"}) * 100 > 80\n          for: 2m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch Heap Usage warning (instance {{ $labels.instance }})\n            description: \"The heap usage is over 80%\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchDiskOutOfSpace\n          expr: elasticsearch_filesystem_data_available_bytes \/ elasticsearch_filesystem_data_size_bytes * 100 < 10\n          for: 0m\n          labels:\n            severity: critical\n          annotations:\n            summary: Elasticsearch disk out of space (instance {{ $labels.instance }})\n            description: \"The disk usage is over 90%\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchDiskSpaceLow\n          expr: elasticsearch_filesystem_data_available_bytes \/ elasticsearch_filesystem_data_size_bytes * 100 < 20\n          for: 2m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch disk space low (instance {{ $labels.instance }})\n            description: \"The disk usage is over 80%\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchClusterRed\n          expr: elasticsearch_cluster_health_status{color=\"red\"} == 1\n          for: 0m\n          labels:\n            severity: critical\n          annotations:\n            summary: Elasticsearch Cluster Red (instance {{ $labels.instance }})\n            description: \"Elastic Cluster Red status\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchClusterYellow\n          expr: elasticsearch_cluster_health_status{color=\"yellow\"} == 1\n          for: 0m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch Cluster Yellow (instance {{ $labels.instance }})\n            description: \"Elastic Cluster Yellow status\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n#        - alert: ElasticsearchHealthyNodes\n#          expr: elasticsearch_cluster_health_number_of_nodes < 3\n#          for: 0m\n#          labels:\n#            severity: critical\n#          annotations:\n#            summary: Elasticsearch Healthy Nodes (instance {{ $labels.instance }})\n#            description: \"Missing node in Elasticsearch cluster\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n#        - alert: ElasticsearchHealthyDataNodes\n#          expr: elasticsearch_cluster_health_number_of_data_nodes < 3\n#          for: 0m\n#          labels:\n#            severity: critical\n#          annotations:\n#            summary: Elasticsearch Healthy Data Nodes (instance {{ $labels.instance }})\n#            description: \"Missing data node in Elasticsearch cluster\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchRelocatingShards\n          expr: elasticsearch_cluster_health_relocating_shards > 0\n          for: 0m\n          labels:\n            severity: info\n          annotations:\n            summary: Elasticsearch relocating shards (instance {{ $labels.instance }})\n            description: \"Elasticsearch is relocating shards\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchRelocatingShardsTooLong\n          expr: elasticsearch_cluster_health_relocating_shards > 0\n          for: 15m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch relocating shards too long (instance {{ $labels.instance }})\n            description: \"Elasticsearch has been relocating shards for 15min\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchInitializingShards\n          expr: elasticsearch_cluster_health_initializing_shards > 0\n          for: 0m\n          labels:\n            severity: info\n          annotations:\n            summary: Elasticsearch initializing shards (instance {{ $labels.instance }})\n            description: \"Elasticsearch is initializing shards\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchInitializingShardsTooLong\n          expr: elasticsearch_cluster_health_initializing_shards > 3\n          for: 15m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch initializing shards too long (instance {{ $labels.instance }})\n            description: \"Elasticsearch has been initializing shards for 15 min\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchUnassignedShards\n          expr: elasticsearch_cluster_health_unassigned_shards > 0\n          for: 0m\n          labels:\n            severity: critical\n          annotations:\n            summary: Elasticsearch unassigned shards (instance {{ $labels.instance }})\n            description: \"Elasticsearch has unassigned shards\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchPendingTasks\n          expr: elasticsearch_cluster_health_number_of_pending_tasks > 0\n          for: 15m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch pending tasks (instance {{ $labels.instance }})\n            description: \"Elasticsearch has pending tasks. Cluster works slowly.\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\"\n        - alert: ElasticsearchNoNewDocuments\n          expr: increase(elasticsearch_indices_docs{es_data_node=\"true\"}[10m]) < 1\n          for: 0m\n          labels:\n            severity: warning\n          annotations:\n            summary: Elasticsearch no new documents (instance {{ $labels.instance }})\n            description: \"No new documents for 10 min!\\n  VALUE = {{ $value }}\\n  LABELS = {{ $labels }} \"\n<\/code><\/pre>\n
\u767b\u5f55prometheus,\u67e5\u770bRules<\/h4>\n
\"image-20230918233456061\"<\/div><\/p>\n
\u53c2\u8003\uff1a<\/h3>\n
https:\/\/github.com\/prometheus-community\/elasticsearch_exporter#elasticsearch-7x-security-privileges<\/code><\/p>\n","protected":false},"excerpt":{"rendered":"
kube-prometheus \u76d1\u63a7ElasticSearch\u6b65\u9aa4 \u6b64\u65b9\u6cd5\u65e2\u53ef\u4ee5\u76d1\u63a7\u96c6\u7fa4\u5185\u7684ElasticSe […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[443,332,490,353,403],"tags":[365,352,369],"class_list":["post-2769","post","type-post","status-publish","format-standard","hentry","category-centos","category-docker","category-elasticsearch","category-kubernetes","category-prometheus","tag-elasticsearch","tag-k8s","tag-prometheus"],"_links":{"self":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2769","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/comments?post=2769"}],"version-history":[{"count":1,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2769\/revisions"}],"predecessor-version":[{"id":2770,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2769\/revisions\/2770"}],"wp:attachment":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/media?parent=2769"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/categories?post=2769"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/tags?post=2769"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}