From cf6ecf5e4016b76d07dfc63ff1eee55261dbda2e Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 16 Aug 2022 17:36:16 +0530 Subject: [PATCH] 1.3.0 (#255) --- .fossa.yml | 5 +++ .github/workflows/ci_build_test.yaml | 33 +++++++++++++++ Gemfile | 1 + Gemfile.lock | 39 +++++++++++------- VERSION | 2 +- docker/Gemfile | 9 +++-- docker/Gemfile.lock | 49 +++++++++++++---------- fluent-plugin-splunk-hec.gemspec | 2 +- lib/fluent/plugin/out_splunk_hec.rb | 15 ++++++- test/fluent/plugin/out_splunk_hec_test.rb | 30 ++++++++++++++ test/test_helper.rb | 11 +++++ 11 files changed, 153 insertions(+), 43 deletions(-) create mode 100644 .fossa.yml diff --git a/.fossa.yml b/.fossa.yml new file mode 100644 index 0000000..6ad77b5 --- /dev/null +++ b/.fossa.yml @@ -0,0 +1,5 @@ +version: 3 +server: https://app.fossa.com +project: + id: "fluent-plugin-splunk-hec" + team: "TA-Automation" diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 5e1e94f..67be8fb 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -194,3 +194,36 @@ jobs: --splunk-password $CI_SPLUNK_PASSWORD \ --nodes-count $MINIKUBE_NODE_COUNTS\ -p no:warnings -s -n auto + fossa-scan: + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: run fossa anlyze and create report + run: | + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash + fossa analyze --include-unused-deps --debug + fossa report attribution --format text > /tmp/THIRDPARTY + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + - name: upload THIRDPARTY file + uses: actions/upload-artifact@v2 + with: + name: THIRDPARTY + path: /tmp/THIRDPARTY + - name: run fossa test + run: | + fossa test --debug + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + semgrep: + runs-on: ubuntu-latest + name: security-sast-semgrep + if: github.actor != 'dependabot[bot]' + steps: + - uses: actions/checkout@v3 + - name: Semgrep + id: semgrep + uses: returntocorp/semgrep-action@v1 + with: + publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} diff --git a/Gemfile b/Gemfile index edf119f..87cc9cf 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ source 'https://rubygems.org' group :test do gem 'simplecov', require: false + gem 'net-smtp', require: false end git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } diff --git a/Gemfile.lock b/Gemfile.lock index 5d9f8e1..0d2d11b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - fluent-plugin-splunk-hec (1.2.13) - fluentd (>= 1.4) + fluent-plugin-splunk-hec (1.3.0) + fluentd (>= 1.5) multi_json (~> 1.13) net-http-persistent (~> 4.0) openid_connect (~> 1.1.8) @@ -11,9 +11,9 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (7.0.2.3) - activesupport (= 7.0.2.3) - activesupport (7.0.2.3) + activemodel (7.0.3) + activesupport (= 7.0.3) + activesupport (7.0.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -28,13 +28,14 @@ GEM cool.io (1.7.1) crack (0.4.5) rexml + digest (3.1.0) docile (1.4.0) - fluentd (1.14.6) + fluentd (1.15.1) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.5, < 3.0.0) + serverengine (>= 2.3.0, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -46,6 +47,7 @@ GEM httpclient (2.8.3) i18n (1.10.0) concurrent-ruby (~> 1.0) + io-wait (0.2.1) json-jwt (1.13.0) activesupport (>= 4.2) aes_key_wrap @@ -54,10 +56,17 @@ GEM mini_mime (>= 0.1.1) mini_mime (1.1.2) minitest (5.15.0) - msgpack (1.5.1) + msgpack (1.5.4) multi_json (1.15.0) net-http-persistent (4.0.1) connection_pool (~> 2.2) + net-protocol (0.1.2) + io-wait + timeout + net-smtp (0.3.1) + digest + net-protocol + timeout openid_connect (1.1.8) activemodel attr_required (>= 1.0.0) @@ -71,7 +80,7 @@ GEM power_assert (2.0.1) prometheus-client (4.0.0) public_suffix (4.0.6) - rack (2.2.3) + rack (2.2.3.1) rack-oauth2 (1.19.0) activesupport attr_required @@ -80,7 +89,7 @@ GEM rack (>= 2.1.0) rake (13.0.6) rexml (3.2.5) - serverengine (2.2.5) + serverengine (2.3.0) sigdump (~> 0.2.2) sigdump (0.2.4) simplecov (0.21.2) @@ -96,14 +105,15 @@ GEM httpclient (>= 2.4) test-unit (3.5.3) power_assert - tzinfo (2.0.4) + timeout (0.2.0) + tzinfo (2.0.5) concurrent-ruby (~> 1.0) - tzinfo-data (1.2022.1) + tzinfo-data (1.2022.2) tzinfo (>= 1.0.0) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) - validate_url (1.0.13) + validate_url (1.0.15) activemodel (>= 3.0.0) public_suffix webfinger (1.2.0) @@ -114,7 +124,7 @@ GEM crack (>= 0.3.2) hashdiff webrick (1.7.0) - yajl-ruby (1.4.2) + yajl-ruby (1.4.3) PLATFORMS ruby @@ -123,6 +133,7 @@ DEPENDENCIES bundler (~> 2.0) fluent-plugin-splunk-hec! minitest (~> 5.0) + net-smtp rake (>= 12.0) simplecov test-unit (~> 3.0) diff --git a/VERSION b/VERSION index 0b1f1ed..589268e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.13 +1.3.0 \ No newline at end of file diff --git a/docker/Gemfile b/docker/Gemfile index 14ae84f..491a855 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -3,20 +3,21 @@ source 'https://rubygems.org' # This is separate gemfile for building docker image that has all plugins # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies -gem "fluentd", ">=1.14.5" +gem "fluentd", ">=1.15" gem "fluent-plugin-systemd", "=1.0.2" gem "fluent-plugin-concat", "=2.4.0" gem "fluent-plugin-prometheus", "=2.0.2" gem "fluent-plugin-jq", "=0.5.1" -gem "fluent-plugin-kubernetes_metadata_filter", ">=2.9.2" -gem "oj", "=3.11.2" +gem 'kubeclient', git: 'https://github.com/splunk/kubeclient.git', ref: '955ec5b' +gem 'fluent-plugin-kubernetes_metadata_filter', '~> 3.1' +gem "oj", ">=3.11.2" gem 'multi_json', '~> 1.13' gem 'net-http-persistent', '~> 4.0' gem 'openid_connect', '~> 1.1.8' gem 'prometheus-client', '>= 2.1.0' gem 'activesupport', '~> 5.2.4.3' gem 'http_parser.rb', '=0.8.0' -gem "rack", ">= 2.1.4" +gem "rack", "=2.2.3.1" gem "fluent-plugin-record-modifier", ">=2.1" diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index b11d1e0..fe65133 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,8 +1,19 @@ +GIT + remote: https://github.com/splunk/kubeclient.git + revision: 955ec5b62f64b6065b25a7c0c76467e8b6badf17 + ref: 955ec5b + specs: + kubeclient (4.9.3) + http (>= 3.0, < 5.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + PATH remote: gem specs: - fluent-plugin-splunk-hec (1.2.13) - fluentd (>= 1.4) + fluent-plugin-splunk-hec (1.3.0) + fluentd (>= 1.5) multi_json (~> 1.13) net-http-persistent (~> 4.0) openid_connect (~> 1.1.8) @@ -37,8 +48,8 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.9.4) - fluentd (>= 0.14.0, < 1.15) + fluent-plugin-kubernetes_metadata_filter (3.1.0) + fluentd (>= 0.14.0, < 1.16) kubeclient (>= 4.0.0, < 5.0.0) lru_redux fluent-plugin-prometheus (2.0.2) @@ -49,12 +60,12 @@ GEM fluent-plugin-systemd (1.0.2) fluentd (>= 0.14.11, < 2) systemd-journal (~> 1.3.2) - fluentd (1.14.6) + fluentd (1.15.1) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.5, < 3.0.0) + serverengine (>= 2.3.0, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -82,11 +93,6 @@ GEM bindata jsonpath (1.1.0) multi_json - kubeclient (4.9.3) - http (>= 3.0, < 5.0) - jsonpath (~> 1.0) - recursive-open-struct (~> 1.1, >= 1.1.1) - rest-client (~> 2.0) lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) @@ -95,7 +101,7 @@ GEM mime-types-data (3.2022.0105) mini_mime (1.1.2) minitest (5.15.0) - msgpack (1.5.1) + msgpack (1.5.4) multi_json (1.15.0) net-http-persistent (4.0.0) connection_pool (~> 2.2) @@ -113,7 +119,7 @@ GEM webfinger (>= 1.0.1) prometheus-client (2.1.0) public_suffix (4.0.6) - rack (2.2.3) + rack (2.2.3.1) rack-oauth2 (1.19.0) activesupport attr_required @@ -127,7 +133,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - serverengine (2.2.5) + serverengine (2.3.0) sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) @@ -138,9 +144,9 @@ GEM systemd-journal (1.3.3) ffi (~> 1.9) thread_safe (0.3.6) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) - tzinfo-data (1.2022.1) + tzinfo-data (1.2022.2) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext @@ -155,7 +161,7 @@ GEM activesupport httpclient (>= 2.4) webrick (1.7.0) - yajl-ruby (1.4.2) + yajl-ruby (1.4.3) PLATFORMS ruby @@ -164,19 +170,20 @@ DEPENDENCIES activesupport (~> 5.2.4.3) fluent-plugin-concat (= 2.4.0) fluent-plugin-jq (= 0.5.1) - fluent-plugin-kubernetes_metadata_filter (>= 2.9.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.1) fluent-plugin-prometheus (= 2.0.2) fluent-plugin-record-modifier (>= 2.1) fluent-plugin-splunk-hec! fluent-plugin-systemd (= 1.0.2) - fluentd (>= 1.14.5) + fluentd (>= 1.15) http_parser.rb (= 0.8.0) + kubeclient! multi_json (~> 1.13) net-http-persistent (~> 4.0) - oj (= 3.11.2) + oj (>= 3.11.2) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - rack (>= 2.1.4) + rack (= 2.2.3.1) BUNDLED WITH 2.3.11 diff --git a/fluent-plugin-splunk-hec.gemspec b/fluent-plugin-splunk-hec.gemspec index 84cc039..6d0d43e 100644 --- a/fluent-plugin-splunk-hec.gemspec +++ b/fluent-plugin-splunk-hec.gemspec @@ -33,7 +33,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = '>= 2.3.0' - spec.add_runtime_dependency 'fluentd', '>= 1.4' + spec.add_runtime_dependency 'fluentd', '>= 1.5' spec.add_runtime_dependency 'multi_json', '~> 1.13' spec.add_runtime_dependency 'net-http-persistent', '~> 4.0' spec.add_runtime_dependency 'openid_connect', '~> 1.1.8' diff --git a/lib/fluent/plugin/out_splunk_hec.rb b/lib/fluent/plugin/out_splunk_hec.rb index 0b83128..a88900d 100644 --- a/lib/fluent/plugin/out_splunk_hec.rb +++ b/lib/fluent/plugin/out_splunk_hec.rb @@ -9,6 +9,7 @@ require 'openssl' require 'multi_json' require 'net/http/persistent' +require 'zlib' module Fluent::Plugin class SplunkHecOutput < SplunkOutput @@ -96,6 +97,9 @@ class SplunkHecOutput < SplunkOutput desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.' config_param :keep_keys, :bool, default: false + desc 'Indicates if GZIP Compression is enabled.' + config_param :gzip_compression, :bool, default: false + desc 'App name' config_param :app_name, :string, default: "hec_plugin_gem" @@ -321,13 +325,20 @@ def new_connection c.override_headers['Authorization'] = "Splunk #{@hec_token}" c.override_headers['__splunk_app_name'] = "#{@app_name}" c.override_headers['__splunk_app_version'] = "#{@app_version}" - end end def write_to_splunk(chunk) post = Net::HTTP::Post.new @api.request_uri - post.body = chunk.read + if @gzip_compression + post.add_field("Content-Encoding", "gzip") + gzip_stream = Zlib::GzipWriter.new StringIO.new + gzip_stream << chunk.read + post.body = gzip_stream.close.string + else + post.body = chunk.read + end + log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." } log.trace { "POST #{@api} body=#{post.body}" } begin diff --git a/test/fluent/plugin/out_splunk_hec_test.rb b/test/fluent/plugin/out_splunk_hec_test.rb index 6b4c717..3ecfb69 100644 --- a/test/fluent/plugin/out_splunk_hec_test.rb +++ b/test/fluent/plugin/out_splunk_hec_test.rb @@ -60,6 +60,12 @@ it 'should consume chunks on 4xx errors' do expect(create_hec_output_driver('hec_host hec_token').instance.consume_chunk_on_4xx_errors).must_equal true end + it 'should default gzip off' do + expect(create_hec_output_driver('hec_host hec_token').instance.gzip_compression).must_equal false + end + it 'should support enabling gzip' do + expect(create_hec_output_driver('hec_host hec_token', 'gzip_compression true').instance.gzip_compression).must_equal true + end end describe 'hec_host validation' do @@ -357,6 +363,15 @@ end end + describe 'gzip encoding' do + it 'should include gzip header when enabled' do + metrics = [ + ['tag', event_time, { 'cup': 0.5, 'memory': 100 }] + ] + with_stub_hec_gzip(events: metrics, conf: 'data_type metric') + end + end + def with_stub_hec(events:, conf: '') host = 'hec.splunk.com' @driver = create_hec_output_driver("hec_host #{host}", conf) @@ -372,6 +387,21 @@ def with_stub_hec(events:, conf: '') hec_req end + def with_stub_hec_gzip(events:, conf: '') + host = 'hec.splunk.com' + @driver = create_hec_output_driver("hec_host #{host}", 'gzip_compression true', conf) + + hec_req = stub_hec_gzip_request("https://#{host}:8088").with do |r| + yield r.body.split(/(?={)\s*(?<=})/).map { |item| JSON.load item } + end + + @driver.run do + events.each { |evt| @driver.feed *evt } + end + + hec_req + end + def verify_sent_events(conf = '', &blk) event = { 'log' => 'everything is good', diff --git a/test/test_helper.rb b/test/test_helper.rb index f68325c..f88ec3a 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -42,4 +42,15 @@ def stub_hec_request(endpoint) 'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}" }) .to_return(body: '{"text":"Success","code":0}') end + + def stub_hec_gzip_request(endpoint) + stub_request(:post, "#{endpoint}/services/collector") + .with(headers: { + 'Authorization' => "Splunk #{TEST_HEC_TOKEN}", + 'User-Agent' => "fluent-plugin-splunk_hec_out/#{Fluent::Plugin::SplunkHecOutput::VERSION}", + 'Content-Encoding' => "gzip" + }, + ) + .to_return(body: '{"text":"GzipSuccess","code":0}') + end end