Prometheus && Alertmanager + prometheus-webhook-dingtalk 告警

Prometheus

下载地址：https://prometheus.io/download/

安装 Prometheus

# 解压下载包
tar -zxvf prometheus-2.11.1.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/src/prometheus-2.11.1.linux-amd64 /usr/local/prometheus

配置 Prometheus

# 修改 prometheus.yml
###
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 你的 alertmanager 服务器:9093
rule_files:
  - "rules/*.yml"
*** 其余 prometheus 刮取策略就不贴出来了，这部分就是告警所需的模块
###
# 添加告警规则(在 prometheus.yml 同级目录下创建 rules 目录)
mkdir rules
vim alerts.yml
###
# 组告警
groups:
# 组名,报警规则组名称
- name: cert_alerts
  rules:
  - alert: certificateEW
    expr: ceil((probe_ssl_earliest_cert_expiry - time())/86400) < 30
    for: 1m
    labels:
      serverity: warning
    annotations:
      summary: The remaining time of the certificate is insufficient
      console: You might want to Renewal certificate
###
# 配置 prometheus service
vim /etc/systemd/system/prometheus.service
###
ExecStart=/usr/local/prometheus/prometheus --
Documentation=https://prometheus.io/docs/introduction/overview/
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/introduction/overview/After=network.target
[Service]
Restart=on-failure
WorkingDirectory=/usr/local/prometheus/
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
[Install]
WantedBy=multi-user.target
###
# 启动 prometheus
systemctl daemon-reload
systemctl start prometheus

Alertmanager

下载地址：https://prometheus.io/download/

安装

# 解压下载的包
tar -C /usr/local/alertmanager -zxvf alertmanager-0.21.0.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk /usr/local/bin/
cp /usr/local/alertmanager/alertmanager /usr/local/bin/
cp /usr/local/alertmanager/amtool /usr/local/bin/

配置

# 修改 alertmanager.yml
###
global:
# 每一分钟检查一次是否恢复
  resolve_timeout: 1m
route:
# 设置默认接收人
  receiver: 'webhook'
# 组告警等待时间,也就是告警产生后等待 10s
  group_wait: 10s
  #两组告警的间隔时间
  group_interval: 10s
  #重复告警的间隔时间，减少相同微信告警的发送频率
  repeat_interval: 1h
 #采用哪个标签来作为分组依据
  group_by: ['alertname']
  routes:
  - receiver: webhook
    group_wait: 10s
    match:
      team: node
receivers:
- name: 'webhook'
  webhook_configs:
  - url: http://localhost:8060/dingtalk/ops_dingding/send
    send_resolved: true
###
# 配置 prometheus service
vim /etc/systemd/system/alertmanager.service
###
[unit]
Description=altermanager
Documentation=http://prometheus.io/docs/

[Service]
ExecStart=/usr/local/alertmanager/alertmanager --config.file=/usr/local/alertmanager/alertmanager.yml
Restart=on-failure

[Install]
WantedBy=multi-user.target
###
# 启动 alertmanager
systemctl daemon-reload
systemctl start prometheus

prometheus-webhook-dingtalk

下载地址：https://github.com/timonwong/prometheus-webhook-dingtalk/releases/tag/v1.4.0

安装

# 解压下载的包
tar -C /usr/local/prometheus-webhook-dingtalk -zxvf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk /usr/local/bin/

配置

# 配置 prometheus-webhook-dingtalk service
vim /etc/systemd/system/prometheus-webhook-dingtalk.service
###
[Unit]
Description=prometheus-webhook-dingtalk
After=network-online.target

[Service]
Restart=on-failure
ExecStart=/usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=XXXX（这里可以换成自己的webhook）

[Install]
WantedBy=multi-user.target
###
# 启动 prometheus-webhook-dingtalk 告警
systemctl daemon-reload
systemctl start prometheus-webhook-dingtalk

查看钉钉告警：已经正常告警