Logstash深入收集Nginx日志
安装nginx
[root@elkstack03 ~]# yum install -y nginx
## 主配置文件
[root@elkstack03 ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 4096;
include /etc/nginx/mime.types;
default_type application/octet-stream;
include /etc/nginx/conf.d/*.conf;
}
## 子配置文件
[root@elkstack03 ~]# vim /etc/nginx/conf.d/www.conf
server{
listen 80;
server_name _;
root /code;
index index.html;
}
[root@elkstack03 ~]# mkdir /code
[root@elkstack03 ~]# echo 'test nginx' > /code/index.html
[root@elkstack03 ~]# systemctl start nginx
将nginx日志改成Json格式
之前我们讲了tomcat日志,在企业中,修改格式需要与开发商量,但是nginx我们不需要,如果需要原来的格式日志,我们可以将日志输出两份,一份 main
格式,一份Json
格式
http{
...
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"ipaddr":"$remote_addr",'
'"login_user":"$remote_user",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"status":"$status"}';
...
}
[root@elkstack03 conf.d]# vim www.conf
server{
listen 80;
server_name www.zls.com;
root /code;
index index.html;
access_log /var/log/nginx/www.zls.com_access_json.log json;
}
[root@elkstack03 conf.d]# cat /etc/nginx/conf.d/blog.conf
server{
listen 80;
server_name blog.zls.com;
root /blog;
index index.html;
access_log /var/log/nginx/blog.zls.com_access_json.log json;
}
使用Logstash收集nginx日志
[root@elkstack03 conf.d]# cat /etc/logstash/conf.d/nginx_file_es.conf
input{
file{
type => "www.zls.com_access"
path => "/var/log/nginx/www.zls.com_access_json.log"
start_position => "beginning"
}
file{
type => "blog.zls.com_access"
path => "/var/log/nginx/blog.zls.com_access_json.log"
start_position => "beginning"
}
}
filter{
json{
source => "message"
remove_field => ["message"]
}
}
output{
elasticsearch{
hosts => ["10.0.0.81:9200"]
index => "%{type}-%{+yyyy.MM.dd}"
codec => "json"
}
}
[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash --path.data=/var/lib/logstash/nginx -f /etc/logstash/conf.d/nginx_file_es.conf &