-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(splunk): support splunk hec logging plugin (#5819)
- Loading branch information
1 parent
4423509
commit e08ec60
Showing
13 changed files
with
655 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| -- | ||
| -- Licensed to the Apache Software Foundation (ASF) under one or more | ||
| -- contributor license agreements. See the NOTICE file distributed with | ||
| -- this work for additional information regarding copyright ownership. | ||
| -- The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| -- (the "License"); you may not use this file except in compliance with | ||
| -- the License. You may obtain a copy of the License at | ||
| -- | ||
| -- http://www.apache.org/licenses/LICENSE-2.0 | ||
| -- | ||
| -- Unless required by applicable law or agreed to in writing, software | ||
| -- distributed under the License is distributed on an "AS IS" BASIS, | ||
| -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| -- See the License for the specific language governing permissions and | ||
| -- limitations under the License. | ||
| -- | ||
|
|
||
| local core = require("apisix.core") | ||
| local ngx = ngx | ||
| local ngx_now = ngx.now | ||
| local http = require("resty.http") | ||
| local log_util = require("apisix.utils.log-util") | ||
| local bp_manager_mod = require("apisix.utils.batch-processor-manager") | ||
|
|
||
|
|
||
| local DEFAULT_SPLUNK_HEC_ENTRY_SOURCE = "apache-apisix-splunk-hec-logging" | ||
| local DEFAULT_SPLUNK_HEC_ENTRY_TYPE = "_json" | ||
|
|
||
|
|
||
| local plugin_name = "splunk-hec-logging" | ||
| local batch_processor_manager = bp_manager_mod.new(plugin_name) | ||
|
|
||
|
|
||
| local schema = { | ||
| type = "object", | ||
| properties = { | ||
| endpoint = { | ||
| type = "object", | ||
| properties = { | ||
| uri = core.schema.uri_def, | ||
| token = { | ||
| type = "string", | ||
| }, | ||
| channel = { | ||
| type = "string", | ||
| }, | ||
| timeout = { | ||
| type = "integer", | ||
| minimum = 1, | ||
| default = 10 | ||
| } | ||
| }, | ||
| required = { "uri", "token" } | ||
| }, | ||
| ssl_verify = { | ||
| type = "boolean", | ||
| default = true | ||
| }, | ||
| }, | ||
| required = { "endpoint" }, | ||
| } | ||
|
|
||
|
|
||
| local _M = { | ||
| version = 0.1, | ||
| priority = 409, | ||
| name = plugin_name, | ||
| schema = batch_processor_manager:wrap_schema(schema), | ||
| } | ||
|
|
||
|
|
||
| function _M.check_schema(conf) | ||
| return core.schema.check(schema, conf) | ||
| end | ||
|
|
||
|
|
||
| local function get_logger_entry(conf) | ||
| local entry = log_util.get_full_log(ngx, conf) | ||
| return { | ||
| time = ngx_now(), | ||
| host = entry.server.hostname, | ||
| source = DEFAULT_SPLUNK_HEC_ENTRY_SOURCE, | ||
| sourcetype = DEFAULT_SPLUNK_HEC_ENTRY_TYPE, | ||
| event = { | ||
| request_url = entry.request.url, | ||
| request_method = entry.request.method, | ||
| request_headers = entry.request.headers, | ||
| request_query = entry.request.querystring, | ||
| request_size = entry.request.size, | ||
| response_headers = entry.response.headers, | ||
| response_status = entry.response.status, | ||
| response_size = entry.response.size, | ||
| latency = entry.latency, | ||
| upstream = entry.upstream, | ||
| } | ||
| } | ||
| end | ||
|
|
||
|
|
||
| local function send_to_splunk(conf, entries) | ||
| local request_headers = {} | ||
| request_headers["Content-Type"] = "application/json" | ||
| request_headers["Authorization"] = "Splunk " .. conf.endpoint.token | ||
| if conf.endpoint.channel then | ||
| request_headers["X-Splunk-Request-Channel"] = conf.endpoint.channel | ||
| end | ||
|
|
||
| local http_new = http.new() | ||
| http_new:set_timeout(conf.endpoint.timeout * 1000) | ||
| local res, err = http_new:request_uri(conf.endpoint.uri, { | ||
| ssl_verify = conf.ssl_verify, | ||
| method = "POST", | ||
| body = core.json.encode(entries), | ||
| headers = request_headers, | ||
| }) | ||
|
|
||
| if err then | ||
| return false, "failed to write log to splunk, " .. err | ||
| end | ||
|
|
||
| if res.status ~= 200 then | ||
| local body | ||
| body, err = core.json.decode(res.body) | ||
| if err then | ||
| return false, "failed to send splunk, http status code: " .. res.status | ||
| else | ||
| return false, "failed to send splunk, " .. body.text | ||
| end | ||
| end | ||
|
|
||
| return true | ||
| end | ||
|
|
||
|
|
||
| function _M.log(conf, ctx) | ||
| local entry = get_logger_entry(conf) | ||
|
|
||
| if batch_processor_manager:add_entry(conf, entry) then | ||
| return | ||
| end | ||
|
|
||
| local process = function(entries) | ||
| return send_to_splunk(conf, entries) | ||
| end | ||
|
|
||
| batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, process) | ||
| end | ||
|
|
||
|
|
||
| return _M |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| --- | ||
| title: splunk-hec-logging | ||
| --- | ||
|
|
||
| <!-- | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| --> | ||
|
|
||
| ## Summary | ||
|
|
||
| - [**Name**](#name) | ||
| - [**Attributes**](#attributes) | ||
| - [**How To Enable**](#how-to-enable) | ||
| - [**Test Plugin**](#test-plugin) | ||
| - [**Disable Plugin**](#disable-plugin) | ||
|
|
||
| ## Name | ||
|
|
||
| The `splunk-hec-logging` plugin is used to forward the request log of `Apache APISIX` to `Splunk HTTP Event Collector (HEC)` for analysis and storage. After the plugin is enabled, `Apache APISIX` will obtain request context information in `Log Phase` serialize it into [Splunk Event Data format](https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector#Event_metadata) and submit it to the batch queue. When the maximum processing capacity of each batch of the batch processing queue or the maximum time to refresh the buffer is triggered, the data in the queue will be submitted to `Splunk HEC`. | ||
|
|
||
| For more info on Batch-Processor in Apache APISIX please refer to: | ||
| [Batch-Processor](../batch-processor.md) | ||
|
|
||
| ## Attributes | ||
|
|
||
| | Name | Requirement | Default | Description | | ||
| | ----------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ||
| | endpoint | required | | Splunk HEC endpoint configuration info | | ||
| | endpoint.uri | required | | Splunk HEC event collector API | | ||
| | endpoint.token | required | | Splunk HEC authentication token | | ||
| | endpoint.channel | optional | | Splunk HEC send data channel identifier, refer to: [About HTTP Event Collector Indexer Acknowledgment](https://docs.splunk.com/Documentation/Splunk/8.2.3/Data/AboutHECIDXAck) | | ||
| | endpoint.timeout | optional | 10 | Splunk HEC send data timeout, time unit: (seconds) | | ||
| | ssl_verify | optional | true | enable `SSL` verification, option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) | | ||
| | max_retry_count | optional | 0 | max number of retries before removing from the processing pipe line | | ||
| | retry_delay | optional | 1 | number of seconds the process execution should be delayed if the execution fails | | ||
| | buffer_duration | optional | 60 | max age in seconds of the oldest entry in a batch before the batch must be processed | | ||
| | inactive_timeout | optional | 5 | max age in seconds when the buffer will be flushed if inactive | | ||
| | batch_max_size | optional | 1000 | max size of each batch | | ||
|
|
||
| ## How To Enable | ||
|
|
||
| The following is an example of how to enable the `splunk-hec-logging` for a specific route. | ||
|
|
||
| ### Full configuration | ||
|
|
||
| ```shell | ||
| curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' | ||
| { | ||
| "plugins":{ | ||
| "splunk-hec-logging":{ | ||
| "endpoint":{ | ||
| "uri":"http://127.0.0.1:8088/services/collector", | ||
| "token":"BD274822-96AA-4DA6-90EC-18940FB2414C", | ||
| "channel":"FE0ECFAD-13D5-401B-847D-77833BD77131", | ||
| "timeout":60 | ||
| }, | ||
| "buffer_duration":60, | ||
| "max_retry_count":0, | ||
| "retry_delay":1, | ||
| "inactive_timeout":2, | ||
| "batch_max_size":10 | ||
| } | ||
| }, | ||
| "upstream":{ | ||
| "type":"roundrobin", | ||
| "nodes":{ | ||
| "127.0.0.1:1980":1 | ||
| } | ||
| }, | ||
| "uri":"/splunk.do" | ||
| }' | ||
| ``` | ||
|
|
||
| ### Minimize configuration | ||
|
|
||
| ```shell | ||
| curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' | ||
| { | ||
| "plugins":{ | ||
| "splunk-hec-logging":{ | ||
| "endpoint":{ | ||
| "uri":"http://127.0.0.1:8088/services/collector", | ||
| "token":"BD274822-96AA-4DA6-90EC-18940FB2414C" | ||
| } | ||
| } | ||
| }, | ||
| "upstream":{ | ||
| "type":"roundrobin", | ||
| "nodes":{ | ||
| "127.0.0.1:1980":1 | ||
| } | ||
| }, | ||
| "uri":"/splunk.do" | ||
| }' | ||
| ``` | ||
|
|
||
| ## Test Plugin | ||
|
|
||
| * Send request to route configured with the `splunk-hec-logging` plugin | ||
|
|
||
| ```shell | ||
| $ curl -i http://127.0.0.1:9080/splunk.do?q=hello | ||
| HTTP/1.1 200 OK | ||
| ... | ||
| hello, world | ||
| ``` | ||
|
|
||
| * Login to Splunk Dashboard to search and view | ||
|
|
||
|  | ||
|
|
||
| ## Disable Plugin | ||
|
|
||
| Disabling the `splunk-hec-logging` plugin is very simple, just remove the `JSON` configuration corresponding to `splunk-hec-logging`. | ||
|
|
||
| ```shell | ||
| $ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' | ||
| { | ||
| "uri": "/hello", | ||
| "plugins": {}, | ||
| "upstream": { | ||
| "type": "roundrobin", | ||
| "nodes": { | ||
| "127.0.0.1:1980": 1 | ||
| } | ||
| } | ||
| }' | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.