y83.第四章 Prometheus大厂监控体系及实战 -- prometheus告警机制进阶(十四)
发布时间
阅读量：
阅读量
12.prometheus告警机制进阶
12.1 钉钉通知
12.1.1 钉钉群组创建机器人–关键字认证
12.1.1.1 钉钉认证–关键字
root@prometheus2:~# mkdir /data/scripts
root@prometheus2:~# cat /data/scripts/dingding-keywords.sh
#!/bin/bash
#
#**********************************************************************************************
#Author: Raymond
#QQ: 88563128
#Date: 2022-05-30
#FileName: /data/scripts/dingding-keywords.sh
#URL: raymond.blog..net
#Description: The test script
#Copyright (C): 2022 All rights reserved
#*********************************************************************************************
#PHONE=$1
#SUBJECT=$2
MESSAGE=$1
/usr/bin/curl -X "POST" 'https://oapi.dingtalk.com/robot/send?access_token=xxxxxx' \
-H 'Content-Type: application/json' \
-d '{"msgtype": "text",
"text": {
"content": "'${MESSAGE}'"
}
}'
AI助手12.1.1.2 测试发送消息
root@prometheus2:~# bash /data/scripts/dingding-keywords.sh alertname:测试信息
{"errcode":0,"errmsg":"ok"}
AI助手
12.1.1.3 部署webhook-dingtalk
root@prometheus2:/apps# wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v1.4.0/prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
root@prometheus2:/apps# tar xf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
root@prometheus2:/apps# ln -sv /apps/prometheus-webhook-dingtalk-1.4.0.linux-amd64 /apps/prometheus-webhook-dingtalk
'/apps/prometheus-webhook-dingtalk' -> '/apps/prometheus-webhook-dingtalk-1.4.0.linux-amd64'
root@prometheus2:/apps# cat /usr/lib/systemd/system/dingtalk.service
[Unit]
Description=prometheus-webhook-dingtalk
After=network-online.target
[Service]
Restart=on-failure
ExecStart=/apps/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile="webhook=https://oapi.dingtalk.com/robot/send?access_token=xxxxxx"
[Install]
WantedBy=multi-user.target
root@prometheus2:/apps# systemctl enable --now dingtalk
root@prometheus2:/apps# ss -ntl| grep 8060
LISTEN 0 4096 *:8060 *:*
AI助手12.1.1.4 部署alertmanager
root@node2:/apps/alertmanager# vim alertmanager.yml
global:
resolve_timeout: 5m
route:
group_by: ['alertname']
group_wait: 10s
group_interval: 10s
repeat_interval: 2m
receiver: 'webhook'
receivers:
- name: 'webhook'
webhook_configs:
- url: 'http://172.31.2.102:8060/dingtalk/webhook/send'
send_resolved: true
root@node2:/apps/alertmanager# ./amtool check-config alertmanager.yml
Checking 'alertmanager.yml' SUCCESS
Found:
- global config
- route
- 1 inhibit rules
- 2 receivers
- 0 templates
AI助手12.1.1.5 配置prometheus
root@prometheus1:/apps/prometheus# cat file_sd/sd_my_server.json
[
{
"targets": ["172.31.2.181:9100","172.31.2.182:9100","172.31.2.183:9100"]
}
]
root@prometheus1:/apps/prometheus# vim prometheus.yml
...
- job_name: 'file_sd_my_server'
file_sd_configs:
- files:
- /apps/prometheus/file_sd/sd_my_server.json
refresh_interval: 10s
root@prometheus1:/apps/prometheus# systemctl restart prometheus
root@node2:/apps/alertmanager# systemctl restart alertmanager
AI助手
12.1.1.6 验证消息发送
12.1.1.7 钉钉验证消息
12.1.2 钉钉群组创建机器人–加签认证
12.1.2.1 加签认证–获取认证
root@prometheus2:~# vim /data/scripts/dingding-label-sign.py
#!/usr/bin/python3
import time
import hmac
import hashlib
import base64
import urllib.parse
timestamp = str(round(time.time() * 1000))
secret = 'xxxxxx' #设置加签的secret
secret_enc = secret.encode('utf-8')
string_to_sign = '{}\n{}'.format(timestamp, secret)
string_to_sign_enc = string_to_sign.encode('utf-8')
hmac_code = hmac.new(secret_enc, string_to_sign_enc, digestmod=hashlib.sha256).digest()
sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
print(timestamp)
print(sign)
root@prometheus2:~# python3 /data/scripts/dingding-label-sign.py
1653910512607
H8EvoSZz75%2Bon5mkGfWcBjShUw43a0CZhUhpMeV%2BwkM%3D
AI助手12.1.2.2 消息发送脚本
root@prometheus2:~# vim /data/scripts/dingding-label-send.sh
#!/bin/bash
#
#**********************************************************************************************
#Author: Raymond
#QQ: 88563128
#Date: 2022-05-30
#FileName: /data/scripts/dingding-label-send.sh
#URL: raymond.blog..net
#Description: The test script
#Copyright (C): 2022 All rights reserved
#*********************************************************************************************
#PHONE=$1
#SUBJECT=$2
MESSAGE=$1
secret='xxxxxx' #设置加签的secret
getkey=$(/usr/bin/python3 /data/scripts/dingding-label-sign.py)
timestamp=${getkey:0:13}
sign=$(echo "${getkey:13:100}" |tr -d '\n')
DateStamp=$(date -d @${getkey:0:10} "+%F %H:%M:%S")
/usr/bin/curl -X "POST" "https://oapi.dingtalk.com/robot/send?access_token=xxxxxx×tamp=${timestamp}&sign=${sign}" \ #token后面自己设置
-H 'Content-Type: application/json' \
-d '{"msgtype": "text",
"text": {
"content": "'${MESSAGE}'"
}
}'
AI助手12.1.2.3 测试发送消息
root@prometheus2:~# bash /data/scripts/dingding-label-send.sh "测试加签认证"
{"errcode":0,"errmsg":"ok"}
AI助手
12.1.2.4 prometheus配置
和上面12.1.1.5 步骤一样
12.1.2.5 webhook-dingtalk启动参数
root@prometheus2:~# systemctl stop dingtalk
#获取当前时间戳和认证秘钥
root@prometheus2:~# python3 /data/scripts/dingding-label-sign.py
1653912164439
VOCKQhGqwAMUMo2ZG4ThL0oNkYbx8lA4AVEvSxmXeMY%3D
#每一个小时要重新获取一次
#基于上一个步骤获取的时间戳和认证信息启动webhook-dingtalk
root@prometheus2:~# /apps/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile="webhook=https://oapi.dingtalk.com/robot/send?access_token=ea8fc7aac8bcbea66639d13ad842681054af55043b20d41faed47d7d42f9c1ba×tamp=1653912164439&sign=VOCKQhGqwAMUMo2ZG4ThL0oNkYbx8lA4AVEvSxmXeMY%3D"
level=info ts=2022-05-30T12:08:46.705Z caller=main.go:62 msg="Starting prometheus-webhook-dingtalk" version="(version=1.4.0, branch=HEAD, revision=02fe8265a98ab4caaa78ebbed209d3f06b87b4a6)"
level=info ts=2022-05-30T12:08:46.705Z caller=main.go:63 msg="Build context" (gogo1.13.5,userroot@eb9f8d8f0437,date20191211-03:00:38)=(MISSING)
level=warn ts=2022-05-30T12:08:46.705Z caller=main.go:105 msg="DEPRECATION: Detected one of the following flags: --ding.profile, --ding.timeout, --template.file"
level=warn ts=2022-05-30T12:08:46.705Z caller=main.go:106 msg="DEPRECATION: Now working in compatibility mode, please consider upgrading your configurations"
level=info ts=2022-05-30T12:08:46.705Z caller=main.go:117 component=configuration msg="Loading templates" templates=
ts=2022-05-30T12:08:46.706Z caller=main.go:133 component=configuration msg="Webhook urls for prometheus alertmanager" urls=http://localhost:8060/dingtalk/webhook/send
level=info ts=2022-05-30T12:08:46.706Z caller=web.go:210 component=web msg="Start listening for connections" address=:8060
#如果不更换时间戳和认证秘钥会出现下面的400报错
level=error ts=2022-05-30T15:18:44.861Z caller=dingtalk.go:103 component=web target=webhook msg="Failed to send notification to DingTalk" respCode=310000 respMsg="invalid timestamp, more: [https://ding-doc.dingtalk.com/doc#/serverapi2/qf2nxq]"
level=info ts=2022-05-30T15:18:44.861Z caller=entry.go:22 component=web http_scheme=http http_proto=HTTP/1.1 http_method=POST remote_addr=172.31.2.182:54418 user_agent=Alertmanager/0.24.0 uri=http://172.31.2.102:8060/dingtalk/webhook/send resp_status=400 resp_bytes_length=27 resp_elapsed_ms=229.918202 msg="request complete"
AI助手12.1.2.6 prometheus测试报警信息
全部评论 (0)
还没有任何评论哟~