promethues常见告警规则

# vim /usr/local/prometheus/rules/node-exporter.yml

# 服务器资源告警策略
groups:
- name: 服务器资源监控
rules:
- alert: 内存使用率过高
expr: (node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 95
for: 5m # 告警持续时间，超过这个时间才会发送给alertmanager
labels:
severity: 严重告警
annotations:
summary: "{{ $labels.instance }} 内存使用率过高，请尽快处理！"
description: "{{ $labels.instance }}内存使用率超过95%,当前使用率{{ $value }}%."

- alert: 服务器宕机
expr: up == 0
for: 3m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 服务器宕机，请尽快处理！"
description: "{{$labels.instance}} 服务器延时超过3分钟，当前状态{{ $value }}. "

- alert: CPU高负荷
expr: 100 - (avg by (instance,job)(irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} CPU使用率过高，请尽快处理！"
description: "{{$labels.instance}} CPU使用大于95%，当前使用率{{ $value }}%. "

- alert: 磁盘IO性能
expr: avg(irate(node_disk_io_time_seconds_total[1m])) by(instance,job)* 100 > 90
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流入磁盘IO使用率过高，请尽快处理！"
description: "{{$labels.instance}} 流入磁盘IO大于90%,当前使用率{{ $value }}%."


- alert: 网络流入
expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流入网络带宽过高，请尽快处理！"
description: "{{$labels.instance}} 流入网络带宽持续5分钟高于100M. RX带宽使用量{{$value}}."

- alert: 网络流出
expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
for: 5m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.instance}} 流出网络带宽过高,请尽快处理！"
description: "{{$labels.instance}} 流出网络带宽持续5分钟高于100M. RX带宽使用量{$value}}."

- alert: TCP连接数
expr: node_netstat_Tcp_CurrEstab > 9000
for: 2m
labels:
severity: 严重告警
annotations:
summary: " TCP_ESTABLISHED过高！"
description: "{{$labels.instance}} TCP_ESTABLISHED大于90%,当前使用率{{ $value }}%."

- alert: 磁盘容量
expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 95
for: 1m
labels:
severity: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 磁盘分区使用率过高，请尽快处理！"
description: "{{$labels.instance}} 磁盘分区使用大于95%，当前使用率{{ $value }}%."

- alert: 平台端口异常告警
expr: probe_success == 0
for: 1m
labels:
severity: 严重告警
annotations:
summary: port down (instance {{ $labels.instance }})
description: "saas平台端口异常
端口状态 == {{ $value }} down
端口服务 == {{ $labels.tag }}"