![\"image-20230619161956800\"](\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\")
<\/div><\/p>\n
\u4fee\u6539\u955c\u50cf\u6e90<\/h3>\n
\u56e0\u4e3a\u751f\u4ea7\u73af\u5883\u662f\u5185\u7f51\u73af\u5883\uff0c\u9700\u8981\u63d0\u524d\u51c6\u5907\u597d\u955c\u50cf\uff0c\u5e76\u4e14\u653e\u5230harbor\u955c\u50cf\u4ed3\u5e93\u4e2d\u3002<\/p>\n
\u6267\u884c:grep -rn 'image: ' *<\/code>\u5373\u53ef\u67e5\u770b\u9700\u8981\u4fee\u6539\u7684\u955c\u50cf\u5185\u5bb9<\/p>\n<\/div><\/p>\n
\u4fee\u6539\u7c7b\u578b\u4e3aNodePort<\/h3>\n
\u4e3a\u4e86\u53ef\u4ee5\u4ece\u5916\u90e8\u8bbf\u95eeprometheus\uff0calertmanager\uff0cgrafana<\/code>\uff0c\u6211\u4eec\u8fd9\u91cc\u4fee\u6539promethes\uff0calertmanager\uff0cgrafana<\/code>\u7684service\u7c7b\u578b\u4e3aNodePort\u7c7b\u578b\u3002<\/p>\n\u4fee\u6539prometheus\u7684service<\/h4>\n
\u4fee\u6539prometheus-service.yaml<\/code><\/p>\n<\/div><\/p>\n
\u4fee\u6539grafana\u7684service<\/h4>\n
\u4fee\u6539grafana-service.yaml<\/code><\/p>\n<\/div><\/p>\n
\u4fee\u6539alertmanager\u7684service<\/h4>\n
\u4fee\u6539alertmanager-service.yaml<\/code><\/p>\n<\/div><\/p>\n
\u5b89\u88c5CRD\u548cprometheus-operator<\/h3>\n\n CustomResourceDefinition\uff1a\u81ea\u5b9a\u4e49\u8d44\u6e90\n<\/p><\/blockquote>\n
\u8fdb\u5165manifests<\/code>\u6587\u4ef6\u8def\u5f84<\/p>\n\u6267\u884c:kubectl apply -f setup\/<\/code><\/p>\n\u5b89\u88c5prometheus, alertmanager, grafana, kube-state-metrics, node-exporter\u7b49\u8d44\u6e90<\/h3>\n
\u6267\u884c:kubectl apply -f .<\/code><\/p>\n\u7a0d\u7b49\u4e00\u6bb5\u65f6\u95f4\u518d\u67e5\u770b\uff0c\u67e5\u770b\u547d\u540d\u7a7a\u95f4monitoring\u4e0b\u9762\u7684pod\u72b6\u6001\uff0c\u76f4\u5230monitoring\u547d\u540d\u7a7a\u95f4\u4e0b\u6240\u6709pod\u90fd\u53d8\u4e3arunning\u72b6\u6001\uff0c\u5c31\u5927\u529f\u544a\u6210\u4e86\u3002<\/p>\n
[root@k8s-m01 manifests]# kubectl get pod -n monitoring\nNAME READY STATUS RESTARTS AGE\nalertmanager-main-0 2\/2 Running 0 3d23h\nalertmanager-main-1 2\/2 Running 0 3d23h\nalertmanager-main-2 2\/2 Running 0 3d23h\nblackbox-exporter-7c75745887-s6qr9 3\/3 Running 0 3d23h\ngrafana-d6c4d689c-qtsf2 1\/1 Running 0 3d23h\nkube-state-metrics-558c88dcf8-lclbq 3\/3 Running 0 3d23h\nnode-exporter-2qqrn 2\/2 Running 0 3d23h\nnode-exporter-c6dbd 2\/2 Running 0 3d23h\nnode-exporter-llwlt 2\/2 Running 0 3d23h\nnode-exporter-mhc27 2\/2 Running 0 3d23h\nnode-exporter-tllch 2\/2 Running 0 3d23h\nprometheus-adapter-668bb8b6d-cs2jz 1\/1 Running 0 3d22h\nprometheus-adapter-668bb8b6d-gswtb 1\/1 Running 0 3d22h\nprometheus-k8s-0 2\/2 Running 0 7h17m\nprometheus-operator-69f74db686-247kx 2\/2 Running 0 3d23h\n<\/code><\/pre>\n\u7ec4\u4ef6\u8bf4\u660e<\/h4>\n\n- node_exporter<\/strong>\uff1a\u7528\u6765\u76d1\u63a7\u8fd0\u7b97\u8282\u70b9\u4e0a\u7684\u5bbf\u4e3b\u673a\u7684\u8d44\u6e90\u4fe1\u606f\uff0c\u9700\u8981\u90e8\u7f72\u5230\u6240\u6709\u8fd0\u7b97\u8282\u70b9<\/li>\n
- kube-state-metric<\/strong>\uff1aprometheus\u91c7\u96c6k8s\u8d44\u6e90\u6570\u636e\u7684exporter\uff0c\u80fd\u591f\u91c7\u96c6\u7edd\u5927\u591a\u6570k8s\u5185\u7f6e\u8d44\u6e90\u7684\u76f8\u5173\u6570\u636e\uff0c\u4f8b\u5982pod\u3001deploy\u3001service\u7b49\u7b49\u3002\u540c\u65f6\u5b83\u4e5f\u63d0\u4f9b\u81ea\u5df1\u7684\u6570\u636e\uff0c\u4e3b\u8981\u662f\u8d44\u6e90\u91c7\u96c6\u4e2a\u6570\u548c\u91c7\u96c6\u53d1\u751f\u7684\u5f02\u5e38\u6b21\u6570\u7edf\u8ba1<\/li>\n
- blackbox_exporter<\/strong>\uff1a\u76d1\u63a7\u4e1a\u52a1\u5bb9\u5668\u5b58\u6d3b\u6027<\/li>\n
- prometheus-adapter<\/strong>\uff1a\u7531\u4e8e\u672c\u8eabprometheus\u5c5e\u4e8e\u7b2c\u4e09\u65b9\u7684 \u89e3\u51b3\u65b9\u6848\uff0c\u539f\u751f\u7684k8s\u7cfb\u7edf\u5e76\u4e0d\u80fd\u5bf9Prometheus\u7684\u81ea\u5b9a\u4e49\u6307\u6807\u8fdb\u884c\u89e3\u6790\uff0c\u5c31\u9700\u8981\u501f\u52a9\u4e8ek8s-prometheus-adapter\u5c06\u8fd9\u4e9b\u6307\u6807\u6570\u636e\u67e5\u8be2\u63a5\u53e3\u8f6c\u6362\u4e3a\u6807\u51c6\u7684Kubernetes\u81ea\u5b9a\u4e49\u6307\u6807\u3002<\/li>\n<\/ul>\n
\u9a8c\u8bc1<\/h3>\n
<\/div><\/p>\n
\u6d4f\u89c8\u5668\u8bbf\u95ee<\/strong><\/h4>\nprometheus\uff1ahttp:\/\/vip:30090\/<\/p>\n
grafana\uff1ahttp:\/\/vip:30091\/login #\u9ed8\u8ba4\u8d26\u53f7\/\u5bc6\u7801\uff1aadmin\/admin<\/p>\n
alertmanager\uff1ahttp:\/\/vip:30092\/<\/p>\n
\u6570\u636e\u6301\u4e45\u5316<\/h2>\n
\u4f7f\u7528kube-prometheus<\/code>\u5b89\u88c5\u7684prometheus\uff0c\u9ed8\u8ba4\u662f\u6ca1\u6709\u5c06\u6570\u636e\u6301\u4e45\u5316\u7684\uff0c\u4e5f\u662f\u5c31\u662fpod \u91cd\u542f\u6570\u636e\u5c31\u4e22\u5931\u4e86\uff0c\u56e0\u6b64kube-prometheus\u6570\u636e\u5fc5\u987b\u505a\u6301\u4e45\u5316<\/p>\n\n \u9ed8\u8ba4\u4e3aNFS \u5b58\u50a8\u7684 StorageClass \u914d\u7f6e<\/strong>\n<\/p><\/blockquote>\nprometheus\u6301\u4e45\u5316\u8bbe\u7f6e<\/h3>\n
prometheus-server \u9ed8\u8ba4\u60c5\u51b5\u4e0b\u6ca1\u6709\u914d\u7f6e\u6570\u636e\u6301\u4e45\u5316\u3002\u6240\u4ee5\u6211\u4eec\u9700\u8981\u505a\u6301\u4e45\u5316\u5b58\u50a8\u3002<\/em><\/p>\n\u5f53\u521d\u521b\u5efakube-prometheus\u7684yaml\u6587\u4ef6\u91cc\u9762\u5e76\u6ca1\u6709\u521b\u5efaStatefulSet\u8d44\u6e90\u3002\u5b98\u65b9\u5b9a\u4e49\u4e86\u4e00\u79cd\u53eb\u505aprometheus\u7684\u8d44\u6e90\uff0c\u8be5\u8d44\u6e90\u521b\u5efa\u4e86StatefulSet<\/p>\n
\u4fee\u6539prometheus-prometheus.yaml<\/code>\u6587\u4ef6<\/h4>\n<\/div><\/p>\n
Grafana\u6301\u4e45\u5316\u8bbe\u7f6e<\/h3>\n\u521b\u5efa grafana-pvc.yaml \u6587\u4ef6<\/strong><\/h4>\n\u7531\u4e8e Grafana \u662f\u90e8\u7f72\u6a21\u5f0f\u4e3a Deployment\uff0c\u6240\u4ee5\u6211\u4eec\u63d0\u524d\u4e3a\u5176\u521b\u5efa\u4e00\u4e2a grafana-pvc.yaml<\/code>\u6587\u4ef6<\/p>\nkind: PersistentVolumeClaim\napiVersion: v1\nmetadata:\n name: grafana-pvc-data\n namespace: monitoring #---\u6307\u5b9anamespace\u4e3amonitoring\nspec:\n storageClassName: k8s-monitor #---\u6307\u5b9aStorageClass\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 40Gi\n<\/code><\/pre>\n\u7f16\u8f91grafana\u7684yaml\u6587\u4ef6<\/em><\/h4>\n<\/div><\/p>\n
kube-prometheus \u90e8\u7f72\u5b8c\u6210\u540e\u51fa\u73b0\u7684\u544a\u8b66\u95ee\u9898<\/h2>\n
\u5728kube-promethesu\u90e8\u7f72\u5b8c\u6210\u4e4b\u540e\uff0c\u4f1a\u51fa\u73b0KubeControllerManagerDown<\/code>\u3001KubeProxyDown<\/code>\u3001KubeSchedulerDown<\/code>\u3001Watchdog<\/code>\u56db\u4e2a\u544a\u8b66<\/p>\n\n \u56e0\u4e3aK8S\u96c6\u7fa4\u4f7f\u7528rke\u90e8\u7f72\u7684\uff0ck8s\u7ec4\u4ef6\u57fa\u672c\u4e0a\u90fd\u662fdocker\u8dd1\uff0cKubeControllerManagerDown<\/code>\u3001KubeProxyDown<\/code>\u3001KubeSchedulerDown<\/code>\u8fd9\u4e09\u4e2a\u9700\u8981\u989d\u5916\u6dfb\u52a0\u76d1\u63a7\n<\/p><\/blockquote>\nKubeControllerManagerDown\/KubeSchedulerDown\/KubeProxyDown\/\u544a\u8b66<\/strong><\/h3>\n\u8fd9\u4e09\u4e2a\u544a\u8b66\u90fd\u662fPrometheus\u65e0\u6cd5\u53d1\u73b0\u76ee\u6807\u6570\u636e\u5bfc\u81f4\u7684\u544a\u8b66\u3002<\/p>\n
RKE\u90e8\u7f72k8s\u89e3\u51b3<\/h4>\n
\u9700\u8981\u4f7f\u7528pushprox-client<\/code>\u548cpushprox-proxy<\/code>\u505a\u4e2d\u8f6c\u6620\u5c04\uff0c\u5177\u4f53yaml\u6587\u4ef6\u5df2\u7ecf\u505a\u8bb0\u5f55<\/p>\n<\/div><\/p>\n
\u4e8c\u8fdb\u5236\u6216\u8005kubeadm\u90e8\u7f72<\/h4>\n
\u53c2\u8003\uff1ahttps:\/\/blog.tearth.cn\/index.php\/archives\/37\/<\/code><\/p>\nWatchdog<\/h3>\n
\u67e5\u770bWatchdog<\/code>\u7684\u544a\u8b66\u63cf\u8ff0\uff1aThis is an alert meant to ensure that the entire alerting pipeline is functional. This alert is always firing, therefore it should always be firing in Alertmanager and always fire against a receiver. There are integrations with various notification mechanisms that send a notification when this alert is not firing. For example the \"DeadMansSnitch\" integration in PagerDuty.<\/code>\uff0c\u7b80\u5355\u6765\u8bf4\uff0c\u8fd9\u4e2a\u544a\u8b66\u662f\u4e3a\u4e86\u4fdd\u8bc1\u544a\u8b66\u529f\u80fd\u53ef\u4ee5\u6b63\u5e38\u4f7f\u7528\uff0c\u5982\u679c\u8fd9\u4e2a\u544a\u8b66\u6d88\u5931\uff0c\u90a3\u5c31\u8868\u793a\u544a\u8b66\u7cfb\u7edf\u51fa\u4e86\u95ee\u9898\u3002\u5982\u679c\u4e0d\u9700\u8981\u8fd9\u4e2a\u529f\u80fd\uff0c\u53ef\u4ee5\u4fee\u6539prometheus<\/code>\u76ee\u5f55\u4e2d\u7684\u76f8\u5173\u6587\u4ef6kubePrometheus-prometheusRule.yaml<\/code>\uff0c\u6ce8\u91ca\u6216\u5220\u9664\u76f8\u5173\u7684\u5185\u5bb9\u5373\u53ef\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"Kubernetes-\u90e8\u7f72Promehteus\u76d1\u63a7 \u672c\u6b21\u4f7f\u7528kube-prometheus\u4e5f\u5c31\u662f\u7f51\u4e0a\u5e38\u7528\u7684Pr […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[332,402,353],"tags":[351,476],"class_list":["post-2669","post","type-post","status-publish","format-standard","hentry","category-docker","category-grafana","category-kubernetes","tag-kubernetes","tag-promehteus"],"_links":{"self":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2669","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/comments?post=2669"}],"version-history":[{"count":2,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2669\/revisions"}],"predecessor-version":[{"id":2671,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/posts\/2669\/revisions\/2671"}],"wp:attachment":[{"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/media?parent=2669"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/categories?post=2669"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/199604.com\/wp-json\/wp\/v2\/tags?post=2669"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
![\"image-20230619161956800\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619161956800.png\")
<\/div><\/p>\n![\"image-20230619162203338\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619162203338.png\")
<\/div><\/p>\n![\"image-20230619162436406\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619162436406.png\")
<\/div><\/p>\n![\"image-20230619162521890\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619162521890.png\")
<\/div><\/p>\n![\"image-20230619162558267\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619162558267.png\")
<\/div><\/p>\n![\"image-20230619163120969\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619163120969.png\")
<\/div><\/p>\n![\"image-20230619170343191\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619170343191.png\")
<\/div><\/p>\n![\"image-20230619171750943\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619171750943.png\")
<\/div><\/p>\n![\"image-20230619164016850\"](\"https:\/\/qn.199604.com\/typoraImg\/image-20230619164016850.png\")
<\/div><\/p>\n