elk日志收集

elk+redis+filebeat收集多日志部署文档

环境:Centos7.6两台
elk--ip:103.39.232.249
nginx--ip:103.39.232.248

基础环境
关闭防火墙
setenforce 0
systemctl stop firewalld
systemctl disable firewalld
sed -i 's/enforcing/disabled/g' /etc/selinux/config

设置yum源
yum install wget -y
cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/repo/Centos-7.repo
wget http://mirrors.aliyun.com/repo/epel-7.repo
yum -y install epel-release
yum install net-tools tree lrzsz vim-enhanced bzip2-x86_64 -y 

配置jdk环境
mkdir /app
cd /app
wget http://download.zhiannet.com/software/java/jdk-11.0.4_linux-x64_bin.rpm
rpm -ivh jdk-11.0.4_linux-x64_bin.rpm
java -version

修改系统参数(重启生效)
vim /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096


添加下面配置：
echo 'vm.max_map_count=655360' >>/etc/sysctl.conf
并执行命令：
sysctl -p

vi /etc/security/limits.d/90-nproc.conf
* soft nproc 2048

vi /etc/security/limits.d/20-nproc.conf
*          soft    nproc     4096
elk        soft    nproc     4096
root       soft    nproc     unlimited

提前下载软件
cd /app
wget http://download.zhiannet.com/software/elk/centos7/elasticsearch-7.3.2-linux-x86_64.tar.gz
wget http://download.zhiannet.com/software/elk/centos7/filebeat-7.3.2-linux-x86_64.tar.gz
wget http://download.zhiannet.com/software/elk/centos7/kibana-7.3.2-linux-x86_64.tar.gz
wget http://download.zhiannet.com/software/elk/centos7/logstash-7.3.2.tar.gz
wget http://download.zhiannet.com/software/elk/centos7/redis-5.0.7.tar.gz

添加用户
useradd elk

修改安装目录权限
chown elk.elk /app

重启服务器
reboot

安装es
cd /app
tar xf elasticsearch-7.3.2-linux-x86_64.tar.gz -C /usr/local/
mv /usr/local/elasticsearch-7.3.2/ /usr/local/elasticsearch
chown -R elk.elk /usr/local/elasticsearch
su elk

修改配置文件
vim /usr/local/elasticsearch/config/jvm.options
-Xms4g
-Xmx4g
## 根据主机内从调整,一般为主机内存的一半

vim /usr/local/elasticsearch/config/elasticsearch.yml
node.name: node-1
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/logs
bootstrap.memory_lock: true
network.host: 0.0.0.0
http.port: 39200
discovery.seed_hosts: ["127.0.0.1"]
cluster.initial_master_nodes: ["node-1"]
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true

启动es
cd /usr/local/elasticsearch
nohup ./bin/elasticsearch &

此时启动es可能会报错,开启内存锁失败
ERROR: bootstrap checks failed
memory locking requested for elasticsearch process but memory is not locked

解决办法:
vim /etc/security/limits.conf
elk soft memlock unlimited
elk hard memlock unlimited
注意:localhost=主机名

vim /etc/sysctl.conf
vm.swappiness=0

sysctl -p
reboot

然后重启es,成功
su elk
cd /usr/local/elasticsearch
nohup ./bin/elasticsearch &


配置es密码
./bin/elasticsearch-setup-passwords interactive
y
******
******
******
...


gihXmSYLcnYAHhcn
浏览器访问
ip:39200
用户:elastic
密码:******
21ops.com

安装kibana
cd /app
tar xf kibana-7.3.2-linux-x86_64.tar.gz -C /usr/local/
mv /usr/local/kibana-7.3.2-linux-x86_64 /usr/local/kibana

配置kibana
vim /usr/local/kibana/config/kibana.yml
server.port: 35601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:39200"]
elasticsearch.username: "elastic"
elasticsearch.password: "******"

启动kibana
cd /usr/local/kibana/
nohup bin/kibana --allow-root &
#直接nohup ./bin/kibana & 会报错

浏览器访问:
ip:35601
用户:elastic
密码:******

安装redis
yum install -y gcc
tar xf /app/redis-5.0.7.tar.gz -C /usr/local/
mv  /usr/local/redis-5.0.7/ /usr/local/redis
cd /usr/local/redis
make MALLOC=libc && echo $?
cd src/
make install
echo $?

vim redis.conf
将daemonize no 改为 daemonize yes
port  12345
requirepass nGjBwhgriFWrLOM2
bind 0.0.0.0

为了安全起见,不要用root用户起redis
chown -R elk.elk /usr/local/redis
su elk
cd /usr/local/redis/src
./redis-server /usr/local/redis/redis.conf

安装logstash
cd /app
tar xf logstash-7.3.2.tar.gz -C /usr/local
mv /usr/local/logstash-7.3.2 /usr/local/logstash

修改配置
mv config/logstash-sample.conf config/logstash-sample.conf_bak
vim config/logstash-redis.conf
input {
  redis {
    host => "127.0.0.1"
    port => 53289
    password => "21ops.com"
    data_type => "list"
    key => "all_keys"
    db => 0
  }
}
output {
  if [fields][log_source] == 'access' {
    elasticsearch {
      hosts => "127.0.0.1:39200"
      index => "nginx-access-%{+YYYY.MM.dd}"
      user => "elastic"
      password => "21ops.com"
    }
  }  
  if [fields][log_source] == 'error' {
    elasticsearch{
      hosts => "127.0.0.1:39200"
      index => "nginx-error-%{+YYYY.MM.dd}"
      user => "elastic"
      password => "21ops.com"
    }
  }
}

启动logstash
nohup ./bin/logstash -f config/logstash-redis.conf &

nginx端配置
安装nginx,略
日志目录如下:
/data/logs/nginx/access/ip/access.log
/data/logs/nginx/error/ip/error.log

安装filebeat
mkdir /app
cd /app
wget http://download.zhiannet.com/software/elk/centos7/filebeat-7.3.2-linux-x86_64.tar.gz
tar xf filebeat-7.3.2-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/filebeat-7.3.2-linux-x86_64/ /usr/local/filebeat

修改配置文件
vim /usr/loca/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/logs/nginx/access/*/access.log
  fields:
    log_source: access
- type: log
  enabled: true
  paths:
    - /data/logs/nginx/error/*/error.log
  fields:
    log_source: error
output.redis:
  hosts: ["103.39.232.249:53289"]
  password: "21ops.com"
  key: "all_keys"
  db: 0

启动
nohup ./filebeat -c filebeat.yml &

redis做成systemd服务
vim /usr/lib/systemd/system/redis.service
[Unit]
Description=Redis
After=network.target

[Service]
Type=forking
User=elk
Group=elk
PIDFile=/var/run/redis_6379.pid
ExecStart=/usr/local/redis/src/redis-server /usr/local/redis/redis.conf
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

es做成systemd服务
vim /usr/lib/systemd/system/elasticsearch.service
[Unit]
Description=elasticsearch
After=network.target

[Service]
LimitMEMLOCK=infinity
Type=simple
User=elk
Group=elk
LimitNOFILE=100000
LimitNPROC=100000
Restart=no
ExecStart=/usr/local/elasticsearch/bin/elasticsearch
PrivateTmp=true

[Install]
WantedBy=multi-user.target


systemctl daemon-reload
systemctl restart elasticsearch

logstash做成systemd服务
vim /usr/lib/systemd/system/logstash.service
[Unit]
Description=logstash

[Service]
Type=simple
User=root
Group=root
#Environment=JAVA_HOME=/usr/local/jdk
Environment=LS_HOME=/usr/local/logstash
Environment=LS_SETTINGS_DIR=/usr/local/logstash/config/
Environment=LS_PIDFILE=/usr/local/logstash/logstash.pid
Environment=LS_USER=root
Environment=LS_GROUP=root
Environment=LS_GC_LOG_FILE=/usr/local/logstash/logs/gc.log
Environment=LS_OPEN_FILES=16384
Environment=LS_NICE=19
Environment=SERVICE_NAME=logstash
Environment=SERVICE_DESCRIPTION=logstash
ExecStart=/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/logstash-redis.conf
Restart=always
WorkingDirectory=/usr/local/logstash
Nice=19
LimitNOFILE=16384

[Install]
WantedBy=multi-user.target

kibana做成systemd服务
vim /usr/lib/systemd/system/kibana.service
[Unit]
Description=Kibana

[Service]
Type=simple
EnvironmentFile=-/usr/local/kibana/config
ExecStart=/usr/local/kibana/bin/kibana --allow-root
Restart=always
WorkingDirectory=/

[Install]
WantedBy=multi-user.target



filebeat做成systemd服务
vim /usr/lib/systemd/system/filebeat.service 
[Unit]
Description=filebeat

[Service]
Type=simple
EnvironmentFile=-/usr/local/filebeat
ExecStart=/usr/local/filebeat/filebeat -c /usr/local/filebeat/filebeat.yml
Restart=always
WorkingDirectory=/

[Install]
WantedBy=multi-user.target