prometheus 使用 ipmi exporter 增加硬件级别监控

prometheus 监控硬件

安装ipmitool 并加载相应模块

yum install ipmitool freeipmi  -y
modprobe ipmi_msghandler
modprobe ipmi_devintf
modprobe ipmi_poweroff
modprobe ipmi_si
modprobe ipmi_watchdog

下载 ipmi_exporter 源码包

wget https://github.com/soundcloud/ipmi_exporter/releases/download/v1.0.0/ipmi_exporter-v1.0.0.linux-amd64.tar.gz  
tar -xf ipmi_exporter-v1.0.0.linux-amd64.tar.gz   -C /opt/
cd /opt/ipmi_exporter-v1.0.0.linux-amd64/

增加配置文件

cat ipmi_remote.yml
modules:
        10.193.x.x:               #远控卡ip地址
                    user: "root"  #远控卡用户
                    pass: "xxxxxxxxxxxxx"  #远控卡密码
                    # Available collectors are bmc, ipmi, chassis, and dcmi 
                    collectors:
                    - bmc
                    - ipmi
                    - dcmi
                    - chassis
                    # Got any sensors you don't care about? Add them here. 
                    exclude_sensor_ids:
                    - 2
                    - 29
                    - 32

启动ipmi_exporter

./ipmi_exporter  --config.file=/usr/local/ipmi_exporter-v1.0.0.linux-amd64/ipmi_remote.yml  --web.listen-address=:19293 & 

增加prometheus server job 配置

#增加监控ipmi exporter rules 规则
  - "rules/Memory_hardware.yml"
  - "rules/power.yml"
  - "rules/fan.yml"
  - "rules/processor.yml"
  - "rules/harddisk.yml"

#增加主配置文件job
#cat /usr/local/prometheus/prometheus.yml
  - job_name: 'ipmi_exporter'
    file_sd_configs:
    - refresh_interval: 5s  
      files:
      - ./conf.d/ipmi_exporter.json
#cat  /usr/local/prometheus/conf.d/ipmi_exporter.json 
[
{
"targets": ["10.65.x.x:19293"],
"labels": {
"hostname": "lgy-storage-glusterxxx"
}
}
]

增加rules 配置文件

# cd /usr/local/prometheus/rules
# cat Memory_hardware.yml  (内存条监控)
groups:
- name: Memory_hardware
  rules:
  - alert: Memory_hardware error
    expr: ipmi_sensor_state{type="Memory"} == 1
    for: 3m
    labels:
      user: caizh
    annotations:
      summary: "Instance {{ $labels.instance }} 内存硬件警告"
      description: "{{ $labels.instance }} of job {{$labels.job}} 内存硬件警告,当前状态[{{ $value }}]."



# cat power.yml （服务器电源模块监控）
groups:
- name: power status
  rules:
  - alert: power bad
    expr: ipmi_sensor_state{name="Status",type="Power Supply"} == 1
    for: 3m
    labels:
      user: caizh
    annotations:
      summary: "Instance {{ $labels.instance }} 电源坏了"
      description: "{{ $labels.instance }} of job {{$labels.job}} 电源坏了,当前状态[{{ $value }}]."


#  cat fan.yml  （服务器风扇监控）
groups:
- name: fan status
  rules:
  - alert: speed fan bad
    expr: ipmi_fan_speed_state{} == 1
    for: 3m
    labels:
      user: caizh
    annotations:
      summary: "Instance {{ $labels.instance }} 风扇坏了"
      description: "{{ $labels.instance }} of job {{$labels.job}} 风扇坏了,当前状态[{{ $value }}]."


# cat processor.yml （服务器处理器监控）
groups:
- name: Processor
  rules:
  - alert: Processor hardware error
    expr: ipmi_sensor_state{name="Status",type="Processor"} == 1
    for: 3m
    labels:
      user: caizh
    annotations:
      summary: "Instance {{ $labels.instance }} 处理器硬件警告"


#  cat harddisk.yml （硬盘监控，主要是raid 组监控，系统盘和数据盘分开做的raid 组，会有两个参数）
groups:
- name: harddisk
  rules:
  - alert: hard disk bad
    expr: ipmi_sensor_state{type="Drive Slot"} == 1
    for: 3m
    labels:
      user: caizh
    annotations:
      summary: "Instance {{ $labels.instance }} 硬盘坏了"
      description: "{{ $labels.instance }} of job {{$labels.job}} 硬盘坏了,当前状态[{{ $value }}]."