作用
将http请求与响应的内容发到kafka的topic，以json的形式发送存储
配置相关
- log_format为自定义配置字段，添加后，默认的请求响应消息将被覆盖
{
"_meta": {
"disable": false
},
"batch_max_size": 1,
"brokers": [
{
"host": "192.168.10.132",
"port": 9091
},
{
"host": "192.168.10.133",
"port": 9092
},
{
"host": "192.168.10.134",
"port": 9097
}
],
"disable": false,
"include_req_body": false,
"include_resp_body": false,
"kafka_topic": "apisix-logger-test",
"log_format": {
"api": "$request_uri",
"userId": "$http_sub",
"userName": "$http_preferred_username",
"now":"$time_iso8601",
},
"producer_type": "async",
"required_acks": 1
}
$开头表示是系统变量$http_开头的，表示是从请求头里获取- 如果获取字段的内容为空，则不会出现在消息体里
- kafka里存储的消息为以下内容
{
"now": "2024-09-10T02:01:22+00:00",
"route_id": "528226539821597458",
"api": "/pkulaw-chat-gpt-api-4/6.0.0.0/gpt/writer/ping",
}
上面配置，是直接在路由或者全局插件配置中，可以通过apisix-dashboard进行配置，使用log_format对记录的信息进行了设置，我们不建议自定义log_format，因为默认的就是最全的信息
不使用log_format，默认的消息体内容如下
[
{
"client_ip": "192.168.60.136",
"upstream": "10.42.4.236:8080",
"apisix_latency": 7.0000324249268,
"start_time": 1716198064095,
"latency": 16.000032424927,
"request": {
"uri": "/kpi/hello",
"method": "GET",
"size": 1757,
"url": "http://test-apisix.pkulaw.com:9080/kpi/hello",
"headers": {
"accept": "text/html,application/xhtml+xml,application/xml;q\u003d0.9,image/avif,image/webp,*/*;q\u003d0.8",
"connection": "close",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
"cookie": "Hm_lvt_8266968662c086f34b2a3e2ae9014bf8\u003d1715752532; Hm_up_8266968662c086f34b2a3e2ae9014bf8\u003d%7B%22ysx_yhqx_20220602%22%3A%7B%22value%22%3A%220%22%2C%22scope%22%3A1%7D%2C%22ysx_hy_20220527%22%3A%7B%22value%22%3A%2206%22%2C%22scope%22%3A1%7D%2C%22uid_%22%3A%7B%22value%22%3A%22fd9959db-ab28-eb11-b390-00155d3c0709%22%2C%22scope%22%3A1%7D%2C%22ysx_yhjs_20220602%22%3A%7B%22value%22%3A%221%22%2C%22scope%22%3A1%7D%7D; cookieUUID\u003dcookieUUID_1715593171791; Hm_lpvt_8266968662c086f34b2a3e2ae9014bf8\u003d1715852645; CookieId\u003daa4be28db45fcd6c3edd84d304533838; SUB\u003d79781f2d-8b4f-483e-ac31-dd55ada12e21; preferred_username\u003dphone2021090709182413393; session_state\u003dde594c09-5681-47c2-ad76-6a7979ff38cd; sensorsdata2015jssdkcross\u003d%7B%22distinct_id%22%3A%22CPcsuEJK%22%2C%22first_id%22%3A%2218f938551ccfe-0b472eec538364-e505625-2073600-18f938551cde56%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218f938551ccfe-0b472eec538364-e505625-2073600-18f938551cde56%22%7D; sajssdk_2015_cross_new_user\u003d1",
"sec-fetch-dest": "document",
"upgrade-insecure-requests": "1",
"accept-language": "zh-CN,en;q\u003d0.8,en-US;q\u003d0.7,zh;q\u003d0.5,zh-TW;q\u003d0.3,zh-HK;q\u003d0.2",
"sec-fetch-mode": "navigate",
"accept-encoding": "gzip, deflate, br",
"sec-fetch-site": "none",
"sec-fetch-user": "?1",
"host": "test-apisix.pkulaw.com",
"x-forwarded-for": "111.198.143.194"
},
"querystring": {}
},
"response": {
"status": 200,
"headers": {
"server": "APISIX/3.8.0",
"content-type": "application/json",
"x-ratelimit-limit": "5",
"transfer-encoding": "chunked",
"connection": "close",
"date": "Mon, 20 May 2024 09:41:04 GMT",
"x-ratelimit-reset": "60",
"x-ratelimit-remaining": "4"
},
"size": 2064
},
"route_id": "513923429800346372",
"upstream_latency": 9,
"service_id": "",
"server": {
"version": "3.8.0",
"hostname": "apisix-78bcfb45c6-26746"
}
}
]
在log_format的同步添加扩展字段
- 这需要修改kafka-logger的源码了
- 获取user-agent中的操作系统，浏览器等信息，你可以添加下面的方法
local function parse_user_agent(user_agent)
local os, browser, version
-- 检测操作系统
if user_agent:find("Windows") then
os = "Windows"
elseif user_agent:find("Macintosh") then
os = "Mac OS"
elseif user_agent:find("Linux") then
os = "Linux"
elseif user_agent:find("Android") then
os = "Android"
elseif user_agent:find("iPhone") then
os = "iOS"
end
-- 检测浏览器
if user_agent:find("Chrome") then
browser = "Chrome"
version = user_agent:match("Chrome%/(%d+%.%d+)")
elseif user_agent:find("Firefox") then
browser = "Firefox"
version = user_agent:match("Firefox%/(%d+%.%d+)")
elseif user_agent:find("Safari") then
browser = "Safari"
version = user_agent:match("Version%/(%d+%.%d+)")
elseif user_agent:find("MSIE") then
browser = "Internet Explorer"
version = user_agent:match("MSIE (%d+%.%d+)")
elseif user_agent:find("Trident") then
browser = "Internet Explorer"
version = user_agent:match("rv:(%d+%.%d+)")
end
return {
os = os or "Unknown",
browser = browser or "Unknown",
version = version or "Unknown"
}
end
在kafka-logger的_M.log(conf, ctx)方法中，添加浏览器扩展字段
function _M.log(conf, ctx)
local entry
if conf.meta_format == "origin" then
entry = log_util.get_req_original(ctx, conf)
else
entry = log_util.get_log_entry(plugin_name, conf, ctx)
end
-- 添加扩展字段开始
local user_agent = ngx.var.http_user_agent
local info = parse_user_agent(user_agent)
entry.os = info.os
entry.browser=info.browser
entry.version=info.version
-- 添加扩展字段结束
if batch_processor_manager:add_entry(conf, entry) then
return
end
...