filebeat
一、部署
1.基于rpm部署
#下载安装包 https://www.elastic.co/
[[email protected] ~]# ll
-rw-r--r-- 1 root root 34965920 Apr 6 11:19 filebeat-7.17.5-x86_64.rpm
#安装filebeat
[[email protected] ~]# rpm -ivh filebeat-7.17.5-x86_64.rpm
#测试
[[email protected] ~]# filebeat -h
Usage:
filebeat [flags]
filebeat [command]
2.基于二进制部署
#下载安装包
[[email protected] ~]# ll
-rw-r--r-- 1 root root 35414086 Apr 6 11:19 filebeat-7.17.5-linux-x86_64.tar.gz
#解压软件包
[[email protected] ~]# tar xf filebeat-7.17.5-linux-x86_64.tar.gz -C /es/softwares/
#查看软件包
[[email protected] ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# ll
total 114888
-rw-r--r-- 1 root root 3780740 Jun 24 2022 fields.yml
-rwxr-xr-x 1 root root 111653152 Jun 24 2022 filebeat
-rw-r--r-- 1 root root 170451 Jun 24 2022 filebeat.reference.yml
-rw------- 1 root root 8348 Jun 24 2022 filebeat.yml
drwxr-xr-x 3 root root 15 Jun 24 2022 kibana
-rw-r--r-- 1 root root 13675 Jun 24 2022 LICENSE.txt
drwxr-xr-x 76 root root 4096 Jun 24 2022 module
drwxr-xr-x 2 root root 4096 Jun 24 2022 modules.d
-rw-r--r-- 1 root root 1987715 Jun 24 2022 NOTICE.txt
-rw-r--r-- 1 root root 814 Jun 24 2022 README.md
#创建软链接
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# ln -svf /es/softwares/filebeat-7.17.5-linux-x86_64/filebeat /usr/local/sbin/
‘/usr/local/sbin/filebeat’ -> ‘/es/softwares/filebeat-7.17.5-linux-x86_64/filebeat’
#验证filebeat安装是否成功
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -h
Usage:
filebeat [flags]
filebeat [command]
二、filebeat的配置文件
1.基础输入输出
#创建工作目录
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# mkdir config
#编写配置文件
[[email protected] ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/config/
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 01-stdin-to-console.yaml
# 配置filebeat的输入端
filebeat.inputs:
# 指定输入端的类型为标准输入
- type: stdin
# 指定filebeat的输出端为console
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#启动filebeat的实例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/01-stdin-to-console.yaml
#输入111信息
#就会输出标准内容
#filebeat输出内容
2023-04-06T19:51:31.398+0800 INFO [registrar] registrar/registrar.go:109 States Loaded from registrar: 0
2023-04-06T19:51:31.398+0800 INFO [crawler] beater/crawler.go:71 Loading Inputs: 1
2023-04-06T19:51:31.398+0800 INFO [crawler] beater/crawler.go:117 starting input, keys present on the config: [filebeat.inputs.0.type]
2023-04-06T19:51:31.398+0800 INFO [crawler] beater/crawler.go:148 Starting input (ID: 16876905907669988323)
2023-04-06T19:51:31.398+0800 INFO [crawler] beater/crawler.go:106 Loading and starting Inputs completed. Enabled inputs: 1
2023-04-06T19:51:31.398+0800 INFO [stdin.harvester] log/harvester.go:309 Harvester started for paths: [] {"harvester_id": "803eb061-bec7-4947-a4ef-31ab3b53edff"}
1111
{
"@timestamp": "2023-04-06T11:51:36.361Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"name": "elk103.com"
},
"agent": {
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com",
"ephemeral_id": "8f0b3dbc-fa47-475a-80dd-4842d72b120f",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
"name": "elk103.com"
},
"log": {
"offset": 0,
"file": {
"path": ""
}
},
"message": "1111",
"input": {
"type": "stdin"
}
}
注意事项:
filebeat是按行输出的,如果没有换行,是不会输出的
2.filbeat的input插件之tcp案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 02-tcp-to-console.yaml
filebeat.inputs:
# 指定类型为tcp
- type: tcp
# 定义tcp监听的主机和端口
host: 0.0.0.0:8888
# 指定filebeat的输出端为console
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#启动
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/02-tcp-to-console.yaml
#测试
在elk101 nc或者telnet
yum -y install nc telnet
[[email protected] ~]# telnet 10.0.0.103 8888
Trying 10.0.0.103...
Connected to 10.0.0.103.
Escape character is '^]'.
1111
#这个时候,在这里任何输入都会输出到filebeat,要退出,要先关闭filebeat
#filebeat输出内容
{
"@timestamp": "2023-04-06T11:54:17.834Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"log": {
"source": {
"address": "10.0.0.101:49202"
}
},
"input": {
"type": "tcp"
},
"host": {
"name": "elk103.com"
},
"agent": {
"ephemeral_id": "2bda6e16-2c4e-4e79-bfae-317e2fa9998a",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com"
},
"ecs": {
"version": "1.12.0"
},
"message": "1111"
}
[[email protected] ~]# echo "AAAAAAAAAAA" | nc 10.0.0.103 8888
#filebeat输出内容
{
"@timestamp": "2023-04-06T11:55:45.428Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"input": {
"type": "tcp"
},
"agent": {
"ephemeral_id": "a77002f1-85de-4668-b576-e5963e8c043b",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"name": "elk103.com"
},
"message": "AAAAAAAAAAA",
"log": {
"source": {
"address": "10.0.0.101:49204"
}
}
}
3.filbeat的input插件之log案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 03-log-to-console.yaml
filebeat.inputs:
# 指定输入类型是log
- type: log
# 指定文件路径
paths:
- /tmp/test-filebeat/*.log
- /tmp/test-filebeat/*/*.json
# 注意,两个*可以递归匹配
- /tmp/test-filebeat/**/*.exe
# 指定filebeat的输出端为console
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#启动
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/03-log-to-console.yaml
#测试
[[email protected] ~]# mkdir -p /tmp/test-filebeat
/tmp/test-filebeat/*.log
#只有/tmp/test-filebeat/*.log会有输出
/tmp/test-filebeat/*/*.json
#只有/test-filebeat下边的第一个子目录下的*.json会有输出
/tmp/test-filebeat/**/*.txt
#只要是/test-filebeat下边的所有txt都会有输出
4.面试题:
下午4点filebeat崩溃,4-5的数据未收集到ES集群。ES集群只有下午5点以后的数据和4点之前的数据。请问如何找回4-5点数据,请说出思路即可。
从es的data目录下找到data/registry/filebeat/log.json,取出时间戳在4-5点的数据。取出读取到的文件偏移量,然后用python或者go语言进行取数即可
5.input的通用字段案例
filebeat input插件的通用字段(common options):
- enabled:
是否启用该组件,有true和false,默认值为true。当设置为false时,表示该input组件不会被加载执行!
- tags:
给每条数据添加一个tags标签列表。
- fields
给数据添加字段。
- fields_under_root
该值默认值为false,将自定义的字段放在一个"fields"的字段中。若设置为true,则将fields的KEY放在顶级字段中。
- processors:
定义处理器,对源数据进行简单的处理。
参考链接:
https://www.elastic.co/guide/en/beats/filebeat/7.17/defining-processors.html
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 04-input_common_options-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /tmp/test-filebeat/*.log
- /tmp/test-filebeat/*/*.json
- /tmp/test-filebeat/**/*.exe
# 是否启用该类型,默认值为true。
enabled: false
- type: tcp
enabled: true
host: "0.0.0.0:8888"
# 给数据打标签,会在顶级字段多出来多个标签
tags: ["Linux","test"]
# 给数据添加KEY-VALUE类型的字段,默认是放在"fields"中的
fields:
school: school1
class: class01
classroom: room07
ip: 219.141.136.10
port: 13306
# 若设置为true时,则将fields添加的自定义字段放在顶级字段中,默认值为false。
fields_under_root: true
# 定义处理器,过滤指定的数据
processors:
# 删除消息是以linux开头的事件(event)
- drop_event:
when:
regexp:
message: "^linux"
# 消息包含error内容事件(event)就可以删除自定义字段或者tags。无法删除内置的字段.
- drop_fields:
when:
contains:
message: "error"
fields: ["class","tags"]
ignore_missing: false
# 修改字段的名称
- rename:
fields:
# 源字段
- from: "school"
# 目标字段
to: "学校"
- from: "log"
to: "日志"
# 转换数据,将字段的类型转换对应的数据类型,并存放在指定的字段中,本案例将其放在"oldboyedu-linux85"字段中
- convert:
fields:
- {from: "ip", to: "test-filebeat.class07_ip", type: "ip"}
- {from: "port", to: "test-filebeat.class07_port", type: "integer"}
# 指定filebeat的输出端为console
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#启动
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/04-input_common_options-to-console.yaml
#filebeat输出内容
{
"@timestamp": "2023-04-06T12:22:05.171Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"input": {
"type": "tcp"
},
"port": 13306,
"ecs": {
"version": "1.12.0"
},
"agent": {
"ephemeral_id": "c580f7d4-1220-4527-bbdc-3228c4180895",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com"
},
"tags": [
"Linux",
"test"
],
"学校": "school1",
"class": "class01",
"classroom": "room07",
"ip": "219.141.136.10",
"日志": {
"source": {
"address": "10.0.0.101:49206"
}
},
"test-filebeat": {
"class07_port": 13306,
"class07_ip": "219.141.136.10"
},
"message": "test",
"host": {
"name": "elk103.com"
}
}
6.包含指定数据采集,排除指定数据采集及json格式数据采集案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 05-log-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /tmp/test-filebeat/*
# 排除以log结尾的文件
exclude_files: ['\.log$']
# 只采集包含指定信息的数据
# include_lines: ['linux']
# 只要包含特定的数据就不采集该事件(event)
# exclude_lines: ['^linux']
# 将message字段的json数据格式进行解析,并将解析的结果放在顶级字段中
json.keys_under_root: true
# 如果解析json格式失败,则会将错误信息添加为一个"error"字段输出
json.add_error_key: true
# 指定filebeat的输出端为console
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
7.使用filebeat采集nginx日志
7.1搭建nginx环境
#添加yum源
cat > /etc/yum.repos.d/nginx.repo <<'EOF'
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF
#安装nginx
[[email protected] ~]# yum -y install nginx
7.2 使用filebeat采集nginx的json格式日志
#修改nginx的配置文件
[[email protected] ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
#注释掉这几行
# log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#
# access_log /var/log/nginx/access.log main;
log_format nginx_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"SendBytes":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"uri":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"tcp_xff":"$proxy_protocol_addr",'
'"http_user_agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access.log nginx_json;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}
#清空access日志内容
[[email protected] ~]# > /var/log/nginx/access.log
#启动nginx
[[email protected] ~]# systemctl start nginx
#编写配置文件
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat nginx_log_json-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /var/log/nginx/access.log*
#以字段的形式显示
json.keys_under_root: true
#报错的话显示错误信息
json.add_error_key: true
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#启动filebeat
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/nginx_log_json-to-console.yaml
#访问测试
[[email protected] ~]# curl 10.0.0.103
#先查看日志
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat /var/log/nginx/access.log
{"@timestamp":"2023-04-06T20:34:03+08:00","host":"10.0.0.103","clientip":"10.0.0.101","SendBytes":615,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.103","uri":"/index.html","domain":"10.0.0.103","xff":"-","referer":"-","tcp_xff":"-","http_user_agent":"curl/7.29.0","status":"200"}
#查看filebeat输出
{
"@timestamp": "2023-04-06T12:34:08.789Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"xff": "-",
"clientip": "10.0.0.101",
"http_host": "10.0.0.103",
"referer": "-",
"status": "200",
"upstreamhost": "-",
"host": {
"name": "elk103.com"
},
"tcp_xff": "-",
"upstreamtime": "-",
"SendBytes": 615,
"input": {
"type": "log"
},
"agent": {
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com",
"ephemeral_id": "822f8504-f1a4-4247-9955-b47cd3c01ecb",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc"
},
"responsetime": 0,
"domain": "10.0.0.103",
"http_user_agent": "curl/7.29.0",
"uri": "/index.html",
"log": {
"offset": 0,
"file": {
"path": "/var/log/nginx/access.log"
}
},
"ecs": {
"version": "1.12.0"
}
}
8.使用filebeat采集tomcat访问日志和错误日志
8.1 搭建tomcat
#下载tomcat软件包
[[email protected] ~]# ll
-rw-r--r-- 1 root root 11625808 Apr 6 17:19 apache-tomcat-9.0.73.tar.gz
#解压软件包
[[email protected] ~]# tar xf apache-tomcat-9.0.73.tar.gz -C /es/softwares/
#配置环境变量并启动tomcat服务
[[email protected] ~]# cat /etc/profile.d/tomcat.sh
#!/bin/bash
export TOMCAT_HOME=/es/softwares/apache-tomcat-9.0.73
export PATH=$PATH:$TOMCAT_HOME/bin
[[email protected] ~]# source /etc/profile.d/tomcat.sh
8.2 使用filebeat采集tomcat的json格式日志
#查看filebeat输出#配置tomcat日志格式
[[email protected] ~]# cd /es/softwares/apache-tomcat-9.0.73/conf/
[[email protected] /es/softwares/apache-tomcat-9.0.73/conf]# cp server.xml{,.bak}
[[email protected] /es/softwares/apache-tomcat-9.0.73/conf]# vim server.xml
#...(切换到行尾修改,大概是在133-149之间)
<Host name="tomcat.com" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="tomcat.com_access_log" suffix=".txt"
pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t&
quot;,"request":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","
;partner":"%{Referer}i","http_user_agent":"%{User-Agent}i"}"/>
</Host>
#启动tomcat
[[email protected] /es/softwares/apache-tomcat-9.0.73/conf]# catalina.sh start
Using CATALINA_BASE: /es/softwares/apache-tomcat-9.0.73
Using CATALINA_HOME: /es/softwares/apache-tomcat-9.0.73
Using CATALINA_TMPDIR: /es/softwares/apache-tomcat-9.0.73/temp
Using JRE_HOME: /es/softwares/jdk1.8.0_291
Using CLASSPATH: /es/softwares/apache-tomcat-9.0.73/bin/bootstrap.jar:/es/softwares/apache-tomcat-9.0.73/bin/tomcat-juli.jar
Using CATALINA_OPTS:
Tomcat started.
#使用filebeat采集tomcat日志
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat tomcat_log-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /es/softwares/apache-tomcat-9.0.73/logs/tomcat.com_access_log*.txt
json.keys_under_root: true
json.add_error_key: true
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true
#访问测试
[[email protected] ~]# cat /etc/hosts
10.0.0.103 elk103.com tomcat.com
[[email protected] ~]# curl tomcat.com:8080
#查看filebeat输出
{
"@timestamp": "2023-04-06T12:47:13.872Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"SendBytes": 615,
"xff": "-",
"referer": "-",
"domain": "tomcat.com",
"input": {
"type": "log"
},
"clientip": "10.0.0.101",
"uri": "/index.html",
"upstreamtime": "-",
"status": "200",
"upstreamhost": "-",
"http_host": "tomcat.com",
"responsetime": 0,
"ecs": {
"version": "1.12.0"
},
"agent": {
"hostname": "elk103.com",
"ephemeral_id": "822f8504-f1a4-4247-9955-b47cd3c01ecb",
"id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5"
},
"log": {
"offset": 314,
"file": {
"path": "/var/log/nginx/access.log"
}
},
"tcp_xff": "-",
"http_user_agent": "curl/7.29.0",
"host": {
"name": "elk103.com"
}
}
8.3 采集tomcat的错误日志多行匹配案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat tomcat_error_log-to-es.yaml
filebeat.inputs:
- type: log
paths:
- /es/softwares/apache-tomcat-9.0.73/logs/catalina*
multiline.type: pattern
multiline.pattern: '^\d{2}'
multiline.negate: true
multiline.match: after
# 指定输出端为ES集群
output.elasticsearch:
hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]
#测试
[[email protected] /es/softwares/apache-tomcat-9.0.73/bin]# ./catalina.sh stop
#先停止服务
#修改配置文件为错误
#启动服务
[[email protected] /es/softwares/apache-tomcat-9.0.73/bin]# ./catalina.sh start
[[email protected] /es/softwares/apache-tomcat-9.0.73/logs]# tail -20 /es/softwares/apache-tomcat-9.0.73/logs/catalina.out
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)
06-Apr-2023 20:56:49.480 WARNING [main] org.apache.catalina.startup.Catalina.parseServerXml Unable to load server configuration from [/es/softwares/apache-tomcat-9.0.73/conf/server.xml]
org.xml.sax.SAXParseException; systemId: file:/es/softwares/apache-tomcat-9.0.73/conf/server.xml; lineNumber: 146; columnNumber: 12; The content of elements must consist of well-formed character data or markup.
at com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1243)
at com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:644)
at org.apache.tomcat.util.digester.Digester.parse(Digester.java:1535)
at org.apache.catalina.startup.Catalina.parseServerXml(Catalina.java:617)
at org.apache.catalina.startup.Catalina.load(Catalina.java:709)
at org.apache.catalina.startup.Catalina.load(Catalina.java:746)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)
06-Apr-2023 20:56:49.480 SEVERE [main] org.apache.catalina.startup.Catalina.start Cannot start server, server instance is not configured
#测试
GET 10.0.0.101:9200/filebeat-7.17.5-2023.04.06-000001/_search
{
"query":{
"match_phrase":{
"message":"at"
}
},
"_source":["message"]
}
#结果
{
"_index": "filebeat-7.17.5-2023.04.06-000001",
"_type": "_doc",
"_id": "VXWsVocBtBzaHDVNQ2_a",
"_score": 1.0,
"_source": {
"message": "06-Apr-2023 20:56:49.474 SEVERE [main] org.apache.tomcat.util.digester.Digester.fatalError Parse fatal error at line [146] column [12]\n\torg.xml.sax.SAXParseException; systemId: file:/es/softwares/apache-tomcat-9.0.73/conf/server.xml; lineNumber: 146; columnNumber: 12; The content of elements must consist of well-formed character data or markup.\n\t\tat com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.createSAXParseException(ErrorHandlerWrapper.java:204)\n\t\tat com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.fatalError(ErrorHandlerWrapper.java:178)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:400)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:327)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLScanner.reportFatalError(XMLScanner.java:1472)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.startOfMarkup(XMLDocumentFragmentScannerImpl.java:2637)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.next(XMLDocumentFragmentScannerImpl.java:2734)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl.next(XMLDocumentScannerImpl.java:605)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:507)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:867)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:796)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:142)\n\t\tat com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1216)\n\t\tat com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:644)\n\t\tat org.apache.tomcat.util.digester.Digester.parse(Digester.java:1535)\n\t\tat org.apache.catalina.startup.Catalina.parseServerXml(Catalina.java:617)\n\t\tat org.apache.catalina.startup.Catalina.load(Catalina.java:709)\n\t\tat org.apache.catalina.startup.Catalina.load(Catalina.java:746)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\t\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\t\tat java.lang.reflect.Method.invoke(Method.java:498)\n\t\tat org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)\n\t\tat org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)"
}
补充:
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# cat config/tomcat_error_log-to-es.yaml filebeat.inputs: - type: log paths: - /es/softwares/apache-tomcat-9.0.73/logs/catalina* # multiline.type: pattern # multiline.pattern: '^\d{2}' # multiline.negate: true # multiline.match: after # 指定输出端为ES集群 output.elasticsearch: hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] #如果不用多行匹配的结果 GET 10.0.0.101:9200/filebeat-7.17.5-2023.04.06-000001/_search { "query":{ "match_phrase":{ "message":"at" } }, "_source":["message"] }
练习
1.使用filebeat采集ES服务的启动日志写入ES集群
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat es_log-to-es.yaml
filebeat.inputs:
- type: log
paths:
- /es/logs/es7/linux-es.log
# 指定输出端为ES集群
output.elasticsearch:
hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]
2.使用filebeat采集docker日志并写入ES集群;
[root@docker01 /es/softwares/filebeat-7.17.5-linux-x86_64/config]# vim docker_log-to-es.yaml
filebeat.inputs:
- type: docker
containers.ids:
- '2a80ae029e3cd467125fb762dd3fcd81bcfe04b7c8f5a3d81cab502b6d8a54ee'
#指定输出端为ES集群
output.elasticsearch:
hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]
[root@docker01 ~]# docker inspect 2a80ae029e3c
[
{
"Id": "2a80ae029e3cd467125fb762dd3fcd81bcfe04b7c8f5a3d81cab502b6d8a54ee",
9.input多行合并
https://www.elastic.co/guide/en/beats/filebeat/7.17/multiline-examples.html#multiline
#例子1
[beat-logstash-some-name-832-2015.11.28] IndexNotFoundException[no such index]
at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:566)
at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:133)
at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:77)
at org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:75)
#log
multiline.type: pattern
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
#例子2
Exception in thread "main" java.lang.NullPointerException
at com.example.myproject.Book.getTitle(Book.java:16)
at com.example.myproject.Author.getBookTitles(Author.java:25)
at com.example.myproject.Bootstrap.main(Bootstrap.java:14)
#filestream
parsers:
- multiline:
type: pattern
pattern: '^[[:space:]]'
negate: false
match: after
#log
multiline.type: pattern
multiline.pattern: '^[[:space:]]'
multiline.negate: false
multiline.match: after
#例子3
Exception in thread "main" java.lang.IllegalStateException: A book has a null property
at com.example.myproject.Author.getBookIds(Author.java:38)
at com.example.myproject.Bootstrap.main(Bootstrap.java:14)
Caused by: java.lang.NullPointerException
at com.example.myproject.Book.getId(Book.java:22)
at com.example.myproject.Author.getBookIds(Author.java:35)
... 1 more
#filestream
parsers:
- multiline:
type: pattern
pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
negate: false
match: after
#log
multiline.type: pattern
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
multiline.negate: false
multiline.match: after
#例子4
printf ("%10.10ld \t %10.10ld \t %s\
%f", w, x, y, z );
#filestream
parsers:
- multiline:
type: pattern
pattern: '\\$'
negate: false
match: before
#log
multiline.type: pattern
multiline.pattern: '\\$'
multiline.negate: false
multiline.match: before
#例子5
[2015-08-24 11:49:14,389][INFO ][env ] [Letha] using [1] data paths, mounts [[/
(/dev/disk1)]], net usable_space [34.5gb], net total_space [118.9gb], types [hfs]
#filestream
parsers:
- multiline:
type: pattern
pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
#log
multiline.type: pattern
multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
#例子6
[2015-08-24 11:49:14,389] Start new event
[2015-08-24 11:49:14,395] Content of processing something
[2015-08-24 11:49:14,399] End event
#filestream
parsers:
- multiline:
type: pattern
pattern: 'Start new event'
negate: true
match: after
flush_pattern: 'End event'
#log
multiline.type: pattern
multiline.pattern: 'Start new event'
multiline.negate: true
multiline.match: after
multiline.flush_pattern: 'End event'
10.使用filebeat采集docker日志
#安装docker
[[email protected] ~]# ll
-rw-r--r-- 1 root root 112195550 Apr 7 10:03 docker-ce-23_0_1.tar.gz
[[email protected] ~]# tar xf docker-ce-23_0_1.tar.gz
[[email protected] ~]# cd docker-ce-23/
[[email protected] ~/docker-ce-23]# yum -y localinstall *.rpm
#配置docker的镜像加速
[[email protected] ~]# cat /etc/docker/daemon.json
{
"data-root": "/var/lib/docker",
"registry-mirrors": ["https://tuv7rqqq.mirror.aliyuncs.com","https://hub-mirror.c.1com/","https://docker.mirrors.ustc.edu.cn","https://reg-mirror.qiniu.com"]
}
#启动docker
[[email protected] ~]# systemctl enable --now docker
#下载nginx镜像
[[email protected] ~]# docker run -dp 88:80 --name mynginx --restart always nginx:1.22.1-alpine
#下载tomcat镜像
[[email protected] ~]# docker run -dp 89:8080 --name mytomcat --restart always tomcat:jre8-alpine
#使用filebeat采集容器日志
[[email protected] ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/config/
#docker方式采集
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat docker-to-console.yaml
filebeat.inputs:
# 指定输入类型为docker类型
- type: docker
# 指定容器的ID
containers.ids:
- '*'
output.console:
pretty: true
#container采集
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat container-to-console.yaml
filebeat.inputs:
- type: container
paths:
- '/var/lib/docker/containers/*/*.log'
# output.console:
# pretty: true
output.elasticsearch:
hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]
#采集日志
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/container-to-console.yam
11.filebeat的input类型之filestream实战案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat filestream-to-console.yaml
filebeat.inputs:
# 指定类型为filestream,在7.16版本中已经弃用log类型
- type: filestream
enabled: false
paths:
- /tmp/test-filebeat/test1.log
- type: filestream
enabled: false
paths:
- /tmp/test-filebeat/test2.json
# 配置解析
parsers:
# 配置json格式解析
- ndjson:
# 将错误消息记录到error字段中
add_error_key: true
# 如果解析的json格式字段和filebeat内置的顶级字段冲突,则覆盖,默认是不覆盖的。
overwrite_keys: true
# 将message解析的字段放入一个自定义的字段下。若不指定该字段,则默认解析的键值对会在顶级字段.
#target: test-filebeat
- type: filestream
enabled: true
paths:
- /tmp/test-filebeat/demo.log
parsers:
- multiline:
type: count
count_lines: 4
- ndjson:
add_error_key: true
overwrite_keys: true
target: test-filebeat-demo
output.console:
pretty: true
[[email protected] /tmp/test-filebeat]# cat test1.log
1111
2222
3333
4444
[[email protected] /tmp/test-filebeat]# cat test2.json
{"name":"test01","age":"25"}
[[email protected] /tmp/test-filebeat]# cat demo.log
{
"name":"stu01",
"age":18
}
{
"name":"stu02",
"age":20
}
#采集数据
{
"@timestamp": "2023-04-07T12:32:50.283Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"input": {
"type": "filestream"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"name": "elk103.com"
},
"agent": {
"ephemeral_id": "bfd69c53-78b1-4a28-958a-9d7aa967bc6e",
"id": "a7447022-b8dd-47a3-8496-27558631e7c2",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com"
},
"log": {
"file": {
"path": "/tmp/test-filebeat/demo.log"
},
"flags": [
"multiline"
],
"offset": 27
},
"test-filebeat-demo": {
"age": 18,
"name": "stu01"
}
}
{
"@timestamp": "2023-04-07T12:32:50.283Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.17.5"
},
"log": {
"offset": 56,
"file": {
"path": "/tmp/test-filebeat/demo.log"
},
"flags": [
"multiline"
]
},
"test-filebeat-demo": {
"name": "stu02",
"age": 20
},
"input": {
"type": "filestream"
},
"ecs": {
"version": "1.12.0"
},
"host": {
"name": "elk103.com"
},
"agent": {
"id": "a7447022-b8dd-47a3-8496-27558631e7c2",
"name": "elk103.com",
"type": "filebeat",
"version": "7.17.5",
"hostname": "elk103.com",
"ephemeral_id": "bfd69c53-78b1-4a28-958a-9d7aa967bc6e"
}
}
12.将数据写入到本地文件案例
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat stdin-to-file.yaml
filebeat.inputs:
- type: stdin
# 指定输出的类型为本地文件
output.file:
# 指定文件存储的路径
path: "/tmp/test-filebeat"
# 指定文件的名称
filename: stdin.log
#采集数据
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/stdin-to-file.yaml
输入111111
[[email protected] /tmp/test-filebeat]# cat stdin.log
{"@timestamp":"2023-04-07T12:35:04.439Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.17.5"},"log":{"offset":0,"file":{"path":""}},"input":{"type":"stdin"},"ecs":{"version":"1.12.0"},"host":{"name":"elk103.com"},"agent":{"ephemeral_id":"9483e7aa-6983-4ab6-b5fa-26676489aa7b","id":"a7447022-b8dd-47a3-8496-27558631e7c2","name":"elk103.com","type":"filebeat","version":"7.17.5","hostname":"elk103.com"},"message":"11111"}
13.写入数据到ES集群
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat log-to-es.yaml
filebeat.inputs:
- type: filestream
enabled: true
paths:
- /tmp/test-filebeat/shopping.json
parsers:
- multiline:
type: count
count_lines: 7
- ndjson:
add_error_key: true
overwrite_keys: true
# 将日志输出到ES集群
output.elasticsearch:
# 指定ES集群地址
hosts:
- "http://10.0.0.101:9200"
- "http://10.0.0.102:9200"
- "http://10.0.0.103:9200"
# 指定索引
index: "linux-es-shopping-%{+yyyy.MM.dd}"
# 禁用索引声明管理周期,若不禁用则自动忽略自定义索引名称
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "linux-es-shopping"
# 指定索引模板的匹配模式
setup.template.pattern: "linux-es-shopping-*"
# 是否覆盖原有的索引模板
setup.template.overwrite: true
# 设置索引模板
setup.template.settings:
# 指定分片数量为8
index.number_of_shards: 8
# 指定副本数量为0
index.number_of_replicas: 0
#采集数据
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/log-to-es.yaml
建议:
一般先output到控制台,测试正确了,在往es上写
14.将多个数据源写入到ES集群不同索引
[[email protected] /tmp/test-filebeat]# cat testlines3.log
{
"name":"testlines3"
}
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat filestream-to-es.yaml
filebeat.inputs:
- type: filestream
enabled: true
tags: "test2"
paths:
- /tmp/test-filebeat/test2.json
parsers:
- ndjson:
add_error_key: true
- type: filestream
enabled: true
tags: "testlines3"
paths:
- /tmp/test-filebeat/testlines3.log
parsers:
- multiline:
type: count
count_lines: 3
- type: filestream
enabled: true
tags: "demo"
paths:
- /tmp/test-filebeat/demo.log
parsers:
- multiline:
type: count
count_lines: 4
- ndjson:
add_error_key: true
overwrite_keys: true
target: test-filebeat-demo
output.elasticsearch:
hosts:
- "http://10.0.0.101:9200"
- "http://10.0.0.102:9200"
- "http://10.0.0.103:9200"
indices:
- index: "linux-es-test2-%{+yyyy.MM.dd}"
when.contains:
tags: "test2"
- index: "linux-es-testlines3-%{+yyyy.MM.dd}"
when.contains:
tags: "testlines3"
- index: "linux-es-demo-%{+yyyy.MM.dd}"
when.contains:
tags: "demo"
setup.ilm.enabled: false
setup.template.name: "linux-es"
setup.template.pattern: "linux-es-*"
setup.template.overwrite: true
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 0
#采集数据
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/filestream-to-es.yaml
#注意setup.template.name和setup.template.pattern别和现有的冲突
15.filebeat模块使用
#有点不太好使,先弃用吧,修改nginx配置文件格式吧
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/config]# modules-to-logstash.yaml
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
# output.logstash:
# hosts: ["10.0.0.101:7777"]
#
#
output.console:
pretty: true
[[email protected] /oldboyedu/softwares/filebeat-7.17.5-linux-x86_64]# ll modules.d |grep nginx
-rw-r--r-- 1 root root 784 Jun 24 2022 nginx.yml.disablednginx.yml.disabled
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64]# cd modules.d
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/modules.d]# mv nginx.yml.disabled nginx.yml
#直接mv一下,把nginx.yml.disabled 改为nginx.yml
[[email protected] /es/softwares/filebeat-7.17.5-linux-x86_64/modules.d]# cat nginx.yml
# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-module-nginx.html
- module: nginx
# Access logs
access:
enabled: true
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#修改这一行
var.paths: ["/tmp/test-filebeat/access.log"]
# Error logs
error:
enabled: false
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:
# Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs
ingress_controller:
enabled: false
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:
标签:filebeat,log,com,elk103,linux,root,es From: https://www.cnblogs.com/world-of-yuan/p/17503998.html./filebeat modules list # 显示所有模块 ./filebeat modules -h # 显示modules帮助命令 ./filebeat -h # 显示帮助命令 ./filebeat modules enable nginx # 启用指定模块 ./filebeat -e # 前台执行