logstash之grok

nginx匹配示例

nginx日志格式
'$remote_user [$time_local]  $http_x_Forwarded_for $remote_addr  $request $status $upstream_status'
                       '$http_x_forwarded_for'
                       '$upstream_addr '
                       'ups_resp_time: $upstream_response_time '
                       'request_time: $request_time';

nginx日志示例
- [09/May/2023:15:01:31 +0800]  11.20.1.30 38.34.246.127  GET / HTTP/1.1 200 -11.20.1.30- ups_resp_time: - request_time: 0.000

grok匹配
filter {
   grok {
       match => {
         "message" => "%{DATA:remote_user} \[%{HTTPDATE:log_times}\]  %{IPV4:http_x_Forwarded_for} %{IPV4:remote_addr}  %{WORD:request_method} %{DATA:uri} HTTP/%{NUMBER:http_version} %{NUMBER:response_code} %{DATA:upstream_status}%{IPV4:http_x_forwarded_for}%{DATA:upstream_addr} ups_resp_time: %{DATA:ups_resp_time} request_time: %{NUMBER:request_time}"
       } 
   }
}

匹配后数据
{
    "http_x_Forwarded_for" => "11.20.1.30",
                    "host" => "elk3",
                 "message" => "- [09/May/2023:15:01:31 +0800]  11.20.1.30 38.34.246.127  GET / HTTP/1.1 200 -11.20.1.30- ups_resp_time: - request_time: 0.000",
          "request_method" => "GET",
         "upstream_status" => "-",
           "ups_resp_time" => "-",
            "request_time" => "0.000",
             "remote_user" => "-",
               "log_times" => "09/May/2023:15:01:31 +0800",
           "upstream_addr" => "-",
                "@version" => "1",
              "@timestamp" => 2023-05-09T08:12:35.912Z,
            "http_version" => "1.1",
             "remote_addr" => "38.34.246.127",
    "http_x_forwarded_for" => "11.20.1.30",
                     "uri" => "/",
           "response_code" => "200"
}

grok使用格式

%{SYNTAX:SEMANTIC}
%{预定义好的表达式的名字:自定义命名}

内置正则

 1 USERNAME [a-zA-Z0-9._-]+
 2 USER %{USERNAME}
 3 EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
 4 EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
 5 INT (?:[+-]?(?:[0-9]+))
 6 BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
 7 NUMBER (?:%{BASE10NUM})
 8 BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
 9 BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
10 
11 POSINT \b(?:[1-9][0-9]*)\b
12 NONNEGINT \b(?:[0-9]+)\b
13 WORD \b\w+\b
14 NOTSPACE \S+
15 SPACE \s*
16 DATA .*?
17 GREEDYDATA .*
18 QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
19 UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
20 # URN, allowing use of RFC 2141 section 2.3 reserved characters
21 URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+
22 
23 # Networking
24 MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
25 CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
26 WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
27 COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
28 IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
29 IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
30 IP (?:%{IPV6}|%{IPV4})
31 HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
32 IPORHOST (?:%{IP}|%{HOSTNAME})
33 HOSTPORT %{IPORHOST}:%{POSINT}
34 
35 # paths
36 PATH (?:%{UNIXPATH}|%{WINPATH})
37 UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
38 TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
39 WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
40 URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
41 URIHOST %{IPORHOST}(?::%{POSINT:port})?
42 # uripath comes loosely from RFC1738, but mostly from what Firefox
43 # doesn't turn into %XX
44 URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
45 #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
46 URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
47 URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
48 URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
49 
50 # Months: January, Feb, 3, 03, 12, December
51 MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
52 MONTHNUM (?:0?[1-9]|1[0-2])
53 MONTHNUM2 (?:0[1-9]|1[0-2])
54 MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
55 
56 # Days: Monday, Tue, Thu, etc...
57 DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
58 
59 # Years?
60 YEAR (?>\d\d){1,2}
61 HOUR (?:2[0123]|[01]?[0-9])
62 MINUTE (?:[0-5][0-9])
63 # '60' is a leap second in most time standards and thus is valid.
64 SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
65 TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
66 # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
67 DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
68 DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
69 ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
70 ISO8601_SECOND (?:%{SECOND}|60)
71 TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
72 DATE %{DATE_US}|%{DATE_EU}
73 DATESTAMP %{DATE}[- ]%{TIME}
74 TZ (?:[APMCE][SD]T|UTC)
75 DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
76 DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
77 DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
78 DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
79 
80 # Syslog Dates: Month Day HH:MM:SS
81 SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
82 PROG [\x21-\x5a\x5c\x5e-\x7e]+
83 SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
84 SYSLOGHOST %{IPORHOST}
85 SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
86 HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
87 
88 # Shortcuts
89 QS %{QUOTEDSTRING}
90 
91 # Log formats
92 SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
93 
94 # Log Levels
95 LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)