Skip to content

Commit

Permalink
feat(error-log-logger): add clickhouse for error-log-logger (#6256)
Browse files Browse the repository at this point in the history 
Co-authored-by: qizhendong <qizhendong@cmss.chinamobile.com>
  • Loading branch information
@qizhendong1
qizhendong1 and qizhendong authored Feb 28, 2022
1 parent acf7a0c commit b41a2e0
Show file tree
Hide file tree
Showing 4 changed files with 379 additions and 1 deletion.
67 changes: 66 additions & 1 deletion apisix/plugins/error-log-logger.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,17 @@ local metadata_schema = {
service_instance_name = {type="string", default = "APISIX Service Instance"},
},
},
clickhouse = {
type = "object",
properties = {
endpoint_addr = {schema_def.uri_def, default="http://127.0.0.1:8123"},
user = {type = "string", default = "default"},
password = {type = "string", default = ""},
database = {type = "string", default = ""},
logtable = {type = "string", default = ""},
},
required = {"endpoint_addr", "user", "password", "database", "logtable"}
},
host = {schema_def.host_def, description = "Deprecated, use `tcp.host` instead."},
port = {type = "integer", minimum = 0, description = "Deprecated, use `tcp.port` instead."},
tls = {type = "boolean", default = false,
Expand All @@ -75,6 +86,7 @@ local metadata_schema = {
oneOf = {
{required = {"skywalking"}},
{required = {"tcp"}},
{required = {"clickhouse"}},
-- for compatible with old schema
{required = {"host", "port"}}
}
Expand Down Expand Up @@ -214,6 +226,57 @@ local function send_to_skywalking(log_message)
end


local function send_to_clickhouse(log_message)
local err_msg
local res = true
core.log.info("sending a batch logs to ", config.clickhouse.endpoint_addr)

local httpc = http.new()
httpc:set_timeout(config.timeout * 1000)

local entries = {}
for i = 1, #log_message, 2 do
-- TODO Here save error log as a whole string to clickhouse 'data' column.
-- We will add more columns in the future.
table.insert(entries, core.json.encode({data=log_message[i]}))
end

local httpc_res, httpc_err = httpc:request_uri(
config.clickhouse.endpoint_addr,
{
method = "POST",
body = "INSERT INTO " .. config.clickhouse.logtable .." FORMAT JSONEachRow "
.. table.concat(entries, " "),
keepalive_timeout = config.keepalive * 1000,
headers = {
["Content-Type"] = "application/json",
["X-ClickHouse-User"] = config.clickhouse.user,
["X-ClickHouse-Key"] = config.clickhouse.password,
["X-ClickHouse-Database"] = config.clickhouse.database
}
}
)

if not httpc_res then
return false, "error while sending data to clickhouse["
.. config.clickhouse.endpoint_addr .. "] " .. httpc_err
end

-- some error occurred in the server
if httpc_res.status >= 400 then
res = false
err_msg = string.format(
"server returned status code[%s] clickhouse[%s] body[%s]",
httpc_res.status,
config.clickhouse.endpoint_addr.endpoint_addr,
httpc_res:read_body()
)
end

return res, err_msg
end


local function update_filter(value)
local level = log_level[value.level]
local status, err = errlog.set_filter_level(level)
Expand All @@ -230,6 +293,8 @@ end
local function send(data)
if config.skywalking then
return send_to_skywalking(data)
elseif config.clickhouse then
return send_to_clickhouse(data)
end
return send_to_tcp_server(data)
end
Expand All @@ -247,7 +312,7 @@ local function process()
core.log.warn("set log filter failed for ", err)
return
end
if not (config.tcp or config.skywalking) then
if not (config.tcp or config.skywalking or config.clickhouse) then
config.tcp = {
host = config.host,
port = config.port,
Expand Down
24 changes: 24 additions & 0 deletions docs/en/latest/plugins/error-log-logger.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ For more info on Batch-Processor in Apache APISIX please refer.
| skywalking.endpoint_addr | string | optional | http://127.0.0.1:12900/v3/logs | | the http endpoint of Skywalking. |
| skywalking.service_name | string | optional | APISIX | | service name for skywalking reporter |
| skywalking.service_instance_name | String | optional | APISIX Instance Name | | Service instance name for skywalking reporter, set it to `$hostname` to get local hostname directly. |
| clickhouse.endpoint_addr | String | optional | http://127.0.0.1:8213 | | clickhouse HTTP endpoint, default http://127.0.0.1:8213 |
| clickhouse.user | String | optional | default | | clickhouse user |
| clickhouse.password | String | optional | | | clickhouse password |
| clickhouse.database | String | optional | | | clickhouse for error log DB name |
| clickhouse.logtable | String | optional | | | clickhouse for error log table name |
| host | string | optional | | | (`Deprecated`, use `tcp.host` instead) IP address or the Hostname of the TCP server. |
| port | integer | optional | | [0,...] | (`Deprecated`, use `tcp.port` instead) Target upstream port. |
| tls | boolean | optional | false | | (`Deprecated`, use `tcp.tls` instead) Control whether to perform SSL verification. |
Expand Down Expand Up @@ -124,3 +129,22 @@ curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/error-log-logger -H 'X-A
"inactive_timeout": 1
}'
```

## How to set the clickhouse

The plugin sends the error log as a string to the `data` field of the clickhouse table.
*TODO Here save error log as a whole string to clickhouse 'data' column. We will add more columns in the future.*
Step: update the attributes of the plugin

```shell
curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/error-log-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"clickhouse": {
"user": "default",
"password": "a",
"database": "error_log",
"logtable": "t",
"endpoint_addr": "http://127.0.0.1:8123"
}
}'
```
24 changes: 24 additions & 0 deletions docs/zh/latest/plugins/error-log-logger.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ title: error-log-logger
| skywalking.endpoint_addr | string | 可选 | http://127.0.0.1:12900/v3/logs | | Skywalking 的 HTTP endpoint 地址，例如：http://127.0.0.1:12800 |
| skywalking.service_name | string | 可选 | APISIX | | skywalking 上报的 service 名称 |
| skywalking.service_instance_name | String | 可选 | APISIX Instance Name | | skywalking 上报的 service 实例名, 如果期望直接获取本机主机名则设置为 `$hostname` |
| clickhouse.endpoint_addr | String | 可选 | http://127.0.0.1:8213 | | clickhouse 的 HTTP endpoint 地址，例如 http://127.0.0.1:8213 |
| clickhouse.user | String | 可选 | default | | clickhouse 的用户名 |
| clickhouse.password | String | 可选 | | | clickhouse 的密码 |
| clickhouse.database | String | 可选 | | | clickhouse 的用于接收 log 的数据库 |
| clickhouse.logtable | String | 可选 | | | clickhouse 的用于接收 log 的表 |
| host | string | 可选 | | | (`弃用`，替换成`tcp.host`) TCP 服务的IP地址或主机名 |
| port | integer | 可选 | | [0,...] | (`弃用`，替换成`tcp.port`) 目标端口 |
| tls | boolean | 可选 | false | | (`弃用`，替换成`tcp.tls`) 用于控制是否执行SSL验证 |
Expand Down Expand Up @@ -122,3 +127,22 @@ curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/error-log-logger -H 'X-A
"inactive_timeout": 1
}'
```

## 如何设置接收日志的 clickhouse 数据库

插件将 error log 作为一个字符串发送到 clickhouse 表的 `data` 字段。
*TODO 将error log 作为一个字符串保持到clickhouse数据库的data字段，未来我们将会增加更多的字段。*
步骤：更新插件属性

```shell
curl http://127.0.0.1:9080/apisix/admin/plugin_metadata/error-log-logger -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
"clickhouse": {
"user": "default",
"password": "a",
"database": "error_log",
"logtable": "t",
"endpoint_addr": "http://127.0.0.1:8123"
}
}'
```
Loading

0 comments on commit b41a2e0

Please sign in to comment.