From 198ce770d25f296e84c58dbd69bbe580bbd7a7d4 Mon Sep 17 00:00:00 2001 From: Thilo Rusche Date: Sat, 11 Jan 2020 10:16:04 +0000 Subject: [PATCH 1/5] ruby 2.7.0, latest patch levels for lower versions --- .ruby-version | 2 +- .travis.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ruby-version b/.ruby-version index ec1cf33..24ba9a3 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.6.3 +2.7.0 diff --git a/.travis.yml b/.travis.yml index 226d882..c664ee9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: ruby rvm: - - "2.4.5" - - "2.5.3" - - "2.6.2" + - "2.4.9" + - "2.5.7" + - "2.6.5" + - "2.7.0" gemfile: - gemfiles/http3.gemfile - gemfiles/http4.gemfile From 3a97c7ee797b4214d3d59d50182e2d68eac2c1c7 Mon Sep 17 00:00:00 2001 From: Thilo Rusche Date: Sat, 11 Jan 2020 11:09:59 +0000 Subject: [PATCH 2/5] fixed ruby 2.7 deprecations --- spec/adapters/http_base_adapter.rb | 12 +++++++++++- spec/adapters/net_http_adapter.rb | 2 +- spec/adapters/open_uri_adapter.rb | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/spec/adapters/http_base_adapter.rb b/spec/adapters/http_base_adapter.rb index 80cfdf8..110ce80 100644 --- a/spec/adapters/http_base_adapter.rb +++ b/spec/adapters/http_base_adapter.rb @@ -21,7 +21,7 @@ def logs_form_data? def parse_uri(query=false) uri = "#{@protocol}://#{@host}:#{@port}#{@path}" - uri = [uri, URI::encode(@data)].join('?') if query && @data + uri = [uri, safe_query_string(@data)].compact.join('?') if query URI.parse(uri) end @@ -36,4 +36,14 @@ def self.is_libcurl? def self.should_log_headers? true end + + def safe_query_string(data) + return nil unless data + + data.to_s.split('&').map do |pair| + pair.split('=', 2).map do |token| + CGI.escape(token.to_s) + end.join('=') + end.join('&') + end end diff --git a/spec/adapters/net_http_adapter.rb b/spec/adapters/net_http_adapter.rb index b513843..8798fa4 100644 --- a/spec/adapters/net_http_adapter.rb +++ b/spec/adapters/net_http_adapter.rb @@ -3,7 +3,7 @@ class NetHTTPAdapter < HTTPBaseAdapter def send_get_request path = @path - path = [@path, URI::encode(@data)].join('?') if @data + path = [@path, safe_query_string(@data)].compact.join('?') Net::HTTP.get_response(@host, path, @port) end diff --git a/spec/adapters/open_uri_adapter.rb b/spec/adapters/open_uri_adapter.rb index 8e5a526..1ebfcfb 100644 --- a/spec/adapters/open_uri_adapter.rb +++ b/spec/adapters/open_uri_adapter.rb @@ -2,7 +2,7 @@ class OpenUriAdapter < HTTPBaseAdapter def send_get_request - open(parse_uri(true)) # rubocop:disable Security/Open + URI.open(parse_uri(true)) end def expected_response_body From 73bb6343546d4c9e87cba52aed97f0b282c4f25c Mon Sep 17 00:00:00 2001 From: Thilo Rusche Date: Sat, 11 Jan 2020 11:15:32 +0000 Subject: [PATCH 3/5] rubocop --- .rubocop.yml | 4 ++-- .rubocop_todo.yml | 2 +- httplog.gemspec | 4 ++-- lib/httplog/adapters/httpclient.rb | 8 ++----- lib/httplog/http_log.rb | 38 +++++++++++++++--------------- lib/httplog/version.rb | 2 +- spec/adapters/http_base_adapter.rb | 2 +- spec/adapters/net_http_adapter.rb | 1 - spec/lib/http_log_spec.rb | 8 ++----- spec/spec_helper.rb | 4 ++-- spec/support/shared_examples.rb | 2 +- 11 files changed, 33 insertions(+), 42 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 3c7ac94..dce4eb9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,7 @@ inherit_from: ./.rubocop_todo.yml AllCops: - TargetRubyVersion: 2.2 + TargetRubyVersion: 2.4 Exclude: - 'db/**/*' - 'db/schema.rb' @@ -28,7 +28,7 @@ Layout/EmptyLinesAroundClassBody: Layout/EmptyLinesAroundModuleBody: EnforcedStyle: no_empty_lines -Layout/IndentArray: +Layout/FirstArrayElementIndentation: EnforcedStyle: consistent Style/Documentation: diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 06b1662..face84b 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -10,7 +10,7 @@ Metrics/BlockLength: # Offense count: 137 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. # URISchemes: http, https -Metrics/LineLength: +Layout/LineLength: Enabled: false # Offense count: 7 diff --git a/httplog.gemspec b/httplog.gemspec index 36ed664..360bcf5 100644 --- a/httplog.gemspec +++ b/httplog.gemspec @@ -18,11 +18,11 @@ Gem::Specification.new do |gem| of third party gems that don't provide their own log output." gem.files = Dir['lib/**/*.rb'] + - %w(httplog.gemspec README.md CHANGELOG.md) + %w[httplog.gemspec README.md CHANGELOG.md] gem.test_files = `git ls-files -- test/*`.split("\n") gem.require_paths = ['lib'] - gem.required_ruby_version = '>= 2.2' + gem.required_ruby_version = '>= 2.4' gem.add_development_dependency 'ethon', ['~> 0.11'] gem.add_development_dependency 'excon', ['~> 0.60'] diff --git a/lib/httplog/adapters/httpclient.rb b/lib/httplog/adapters/httpclient.rb index 6895c89..89cbbe0 100644 --- a/lib/httplog/adapters/httpclient.rb +++ b/lib/httplog/adapters/httpclient.rb @@ -44,17 +44,13 @@ class Session # it's `create_socket(host, port)` if instance_method(:create_socket).arity == 1 def create_socket(site) - if HttpLog.url_approved?("#{site.host}:#{site.port}") - HttpLog.log_connection(site.host, site.port) - end + HttpLog.log_connection(site.host, site.port) if HttpLog.url_approved?("#{site.host}:#{site.port}") orig_create_socket(site) end else def create_socket(host, port) - if HttpLog.url_approved?("#{host}:#{port}") - HttpLog.log_connection(host, port) - end + HttpLog.log_connection(host, port) if HttpLog.url_approved?("#{host}:#{port}") orig_create_socket(host, port) end end diff --git a/lib/httplog/http_log.rb b/lib/httplog/http_log.rb index b833c9a..302365c 100644 --- a/lib/httplog/http_log.rb +++ b/lib/httplog/http_log.rb @@ -7,7 +7,7 @@ require 'rack' module HttpLog - LOG_PREFIX = '[httplog] '.freeze + LOG_PREFIX = '[httplog] ' PARAM_MASK = '[FILTERED]' class BodyParsingError < StandardError; end @@ -107,9 +107,7 @@ def log_body(body, encoding = nil, content_type = nil) end def parse_body(body, encoding, content_type) - unless text_based?(content_type) - raise BodyParsingError, "(not showing binary data)" - end + raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type) if body.is_a?(Net::ReadAdapter) # open-uri wraps the response in a Net::ReadAdapter that defers reading @@ -145,6 +143,7 @@ def log_data(data) def log_compact(method, uri, status, seconds) return unless config.compact_log + status = Rack::Utils.status_code(status) unless status == /\d{3}/ log("#{method.to_s.upcase} #{masked(uri)} completed with status code #{status} in #{seconds.to_f.round(6)} seconds") end @@ -155,6 +154,7 @@ def transform_response_code(response_code_name) def colorize(msg) return msg unless config.color + if config.color.is_a?(Hash) msg = Rainbow(msg).color(config.color[:color]) if config.color[:color] msg = Rainbow(msg).bg(config.color[:background]) if config.color[:background] @@ -194,26 +194,26 @@ def json_payload(data = {}) if config.compact_log { - method: data[:method].to_s.upcase, - url: masked(data[:url]), + method: data[:method].to_s.upcase, + url: masked(data[:url]), response_code: data[:response_code].to_i, - benchmark: data[:benchmark] + benchmark: data[:benchmark] } else { - method: data[:method].to_s.upcase, - url: masked(data[:url]), - request_body: masked(data[:request_body]), - request_headers: masked(data[:request_headers].to_h), - response_code: data[:response_code].to_i, - response_body: parsed_body, + method: data[:method].to_s.upcase, + url: masked(data[:url]), + request_body: masked(data[:request_body]), + request_headers: masked(data[:request_headers].to_h), + response_code: data[:response_code].to_i, + response_body: parsed_body, response_headers: data[:response_headers].to_h, - benchmark: data[:benchmark] + benchmark: data[:benchmark] } end end - def masked(msg, key=nil) + def masked(msg, key = nil) return msg if config.filter_parameters.empty? return msg if msg.nil? @@ -221,15 +221,15 @@ def masked(msg, key=nil) # in its entirety. return (config.filter_parameters.include?(key.downcase) ? PARAM_MASK : msg) if key - # Otherwise, we'll parse Strings for key=valye pairs... + # Otherwise, we'll parse Strings for key=value pairs... case msg when *string_classes - config.filter_parameters.reduce(msg) do |m,key| - m.to_s.gsub(/(#{key})=[^&]+/i, "#{key}=#{PARAM_MASK}") + config.filter_parameters.reduce(msg) do |m, k| + m.to_s.gsub(/(#{k})=[^&]+/i, "#{k}=#{PARAM_MASK}") end # ...and recurse over hashes when *hash_classes - Hash[msg.map {|k,v| [k, masked(v, k)]}] + Hash[msg.map { |k, v| [k, masked(v, k)] }] else log "*** FILTERING NOT APPLIED BECAUSE #{msg.class} IS UNEXPECTED ***" msg diff --git a/lib/httplog/version.rb b/lib/httplog/version.rb index f740b02..4768556 100644 --- a/lib/httplog/version.rb +++ b/lib/httplog/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HttpLog - VERSION = '1.3.3'.freeze + VERSION = '1.3.3' end diff --git a/spec/adapters/http_base_adapter.rb b/spec/adapters/http_base_adapter.rb index 110ce80..63ae80d 100644 --- a/spec/adapters/http_base_adapter.rb +++ b/spec/adapters/http_base_adapter.rb @@ -19,7 +19,7 @@ def logs_form_data? true end - def parse_uri(query=false) + def parse_uri(query = false) uri = "#{@protocol}://#{@host}:#{@port}#{@path}" uri = [uri, safe_query_string(@data)].compact.join('?') if query URI.parse(uri) diff --git a/spec/adapters/net_http_adapter.rb b/spec/adapters/net_http_adapter.rb index 8798fa4..aef2a88 100644 --- a/spec/adapters/net_http_adapter.rb +++ b/spec/adapters/net_http_adapter.rb @@ -2,7 +2,6 @@ class NetHTTPAdapter < HTTPBaseAdapter def send_get_request - path = @path path = [@path, safe_query_string(@data)].compact.join('?') Net::HTTP.get_response(@host, path, @port) end diff --git a/spec/lib/http_log_spec.rb b/spec/lib/http_log_spec.rb index 95b51a1..9c61641 100644 --- a/spec/lib/http_log_spec.rb +++ b/spec/lib/http_log_spec.rb @@ -91,9 +91,7 @@ def configure it { is_expected.to_not include('Header:') } it { is_expected.to_not include("\e[0") } - unless adapter_class.is_libcurl? - it { is_expected.to include("Connecting: #{host}:#{port}") } - end + it { is_expected.to include("Connecting: #{host}:#{port}") } unless adapter_class.is_libcurl? it { expect(res).to be_a adapter.response if adapter.respond_to? :response } @@ -137,9 +135,7 @@ def configure if adapter_class.method_defined? :send_post_request let!(:res) { adapter.send_post_request } - unless adapter_class.is_libcurl? - it { is_expected.to include("Connecting: #{host}:#{port}") } - end + it { is_expected.to include("Connecting: #{host}:#{port}") } unless adapter_class.is_libcurl? it_behaves_like 'logs request', 'POST' it_behaves_like 'logs expected response' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0cf9122..0fb357a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -17,8 +17,8 @@ require 'loggers/gelf_mock' require 'adapters/http_base_adapter' -Dir[File.dirname(__FILE__) + '/adapters/*.rb'].each { |f| require f } -Dir['./spec/support/**/*.rb'].each { |f| require f } +Dir[File.dirname(__FILE__) + '/adapters/*.rb'].sort.each { |f| require f } +Dir['./spec/support/**/*.rb'].sort.each { |f| require f } # Start a local rack server to serve up test pages. @server_thread = Thread.new do diff --git a/spec/support/shared_examples.rb b/spec/support/shared_examples.rb index be2152a..7c9fe70 100644 --- a/spec/support/shared_examples.rb +++ b/spec/support/shared_examples.rb @@ -70,7 +70,7 @@ end RSpec.shared_examples 'filtered parameters' do - let(:filter_parameters) { %w(foo) } + let(:filter_parameters) { %w[foo] } it 'masks the filtered value' do # is_expected.to include('foo=[FILTERED]&').or exclude('foo') From cd60aeeb493b506c7fb913edf250682f01273785 Mon Sep 17 00:00:00 2001 From: Thilo Rusche Date: Sat, 11 Jan 2020 11:17:47 +0000 Subject: [PATCH 4/5] version bump 1.4.0 --- CHANGELOG.md | 6 ++++++ Gemfile.lock | 4 ++-- lib/httplog/version.rb | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1cbb65..117ff01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.4.0 - TBD + +* Bumped ruby version to 2.7 +* Fixed ruby 2.7 deprecations +* Removed support for ruby < 2.4 + ## 1.3.3 - 2019-11-14 * [#83](https://github.com/trusche/httplog/pull/83) Support for graylog diff --git a/Gemfile.lock b/Gemfile.lock index b9a5878..47b4617 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - httplog (1.3.3) + httplog (1.4.0) rack (>= 1.0) rainbow (>= 2.0.0) @@ -129,4 +129,4 @@ DEPENDENCIES thin (~> 1.7) BUNDLED WITH - 2.0.2 + 2.1.2 diff --git a/lib/httplog/version.rb b/lib/httplog/version.rb index 4768556..6e8b1a2 100644 --- a/lib/httplog/version.rb +++ b/lib/httplog/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HttpLog - VERSION = '1.3.3' + VERSION = '1.4.0' end From 3a9afe56214d99661164774f4bd9bb53841161e1 Mon Sep 17 00:00:00 2001 From: Vladimir Kitaev Date: Tue, 12 Nov 2019 14:47:34 +0300 Subject: [PATCH 5/5] Allow deep mask json and custom JSON serializer. Fix non UTF-8 for json and graylog formats. --- Gemfile.lock | 2 + README.md | 11 ++- httplog.gemspec | 1 + lib/httplog/adapters/ethon.rb | 3 +- lib/httplog/adapters/excon.rb | 3 +- lib/httplog/adapters/http.rb | 8 +- lib/httplog/adapters/httpclient.rb | 8 +- lib/httplog/adapters/net_http.rb | 3 +- lib/httplog/adapters/patron.rb | 3 +- lib/httplog/configuration.rb | 48 ++++++----- lib/httplog/http_log.rb | 99 +++++++++++++++++++---- spec/adapters/net_http_adapter.rb | 2 +- spec/lib/http_log_spec.rb | 124 ++++++++++++++++++++++++----- spec/spec_helper.rb | 3 + spec/support/index.json | 1 + spec/support/shared_examples.rb | 23 +++++- spec/support/test_server.rb | 1 + 17 files changed, 273 insertions(+), 70 deletions(-) create mode 100644 spec/support/index.json diff --git a/Gemfile.lock b/Gemfile.lock index 47b4617..d65a3fc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -70,6 +70,7 @@ GEM notiffany (0.1.3) nenv (~> 0.1) shellany (~> 0.0) + oj (3.9.2) patron (0.13.3) pry (0.12.2) coderay (~> 1.1.0) @@ -122,6 +123,7 @@ DEPENDENCIES httpclient (~> 2.8) httplog! listen (~> 3.0) + oj (>= 3.9.2) patron (~> 0.12) rake (~> 12.3) rspec (~> 3.7) diff --git a/README.md b/README.md index 5be73c2..33a0f22 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,14 @@ HttpLog.configure do |config| config.url_whitelist_pattern = nil config.url_blacklist_pattern = nil - # Mask the values of sensitive requestparameters + # Mask sensitive information in request and response JSON data. + # Enable global JSON masking by setting the parameter to `/.*/` + config.url_masked_body_pattern = nil + + # You can specify any custom JSON serializer that implements `load` and `dump` class methods + config.json_parser = JSON + + # Mask the values of sensitive request parameters config.filter_parameters = %w[password] end ``` @@ -121,7 +128,7 @@ HttpLog.configure do |config| end ``` -If you use Graylog and want to use its search features such as "rounded_benchmark:>1 AND method:PUT", +If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT", you can use this configuration: ```ruby diff --git a/httplog.gemspec b/httplog.gemspec index 360bcf5..8f9d0f5 100644 --- a/httplog.gemspec +++ b/httplog.gemspec @@ -37,6 +37,7 @@ Gem::Specification.new do |gem| gem.add_development_dependency 'rspec', ['~> 3.7'] gem.add_development_dependency 'simplecov', ['~> 0.15'] gem.add_development_dependency 'thin', ['~> 1.7'] + gem.add_development_dependency 'oj', ['>= 3.9.2'] gem.add_dependency 'rack', ['>= 1.0'] gem.add_dependency 'rainbow', ['>= 2.0.0'] diff --git a/lib/httplog/adapters/ethon.rb b/lib/httplog/adapters/ethon.rb index 877784c..ff60d5e 100644 --- a/lib/httplog/adapters/ethon.rb +++ b/lib/httplog/adapters/ethon.rb @@ -39,7 +39,8 @@ def perform response_headers: headers.map { |header| header.split(/:\s/) }.to_h, benchmark: bm, encoding: encoding, - content_type: content_type + content_type: content_type, + mask_body: HttpLog.masked_body_url?(url) ) return_code end diff --git a/lib/httplog/adapters/excon.rb b/lib/httplog/adapters/excon.rb index 95741f9..ee6f651 100644 --- a/lib/httplog/adapters/excon.rb +++ b/lib/httplog/adapters/excon.rb @@ -45,7 +45,8 @@ def request(params, &block) response_headers: headers, benchmark: bm, encoding: headers['Content-Encoding'], - content_type: headers['Content-Type'] + content_type: headers['Content-Type'], + mask_body: HttpLog.masked_body_url?(url) ) result end diff --git a/lib/httplog/adapters/http.rb b/lib/httplog/adapters/http.rb index cfc5bcc..b522cae 100644 --- a/lib/httplog/adapters/http.rb +++ b/lib/httplog/adapters/http.rb @@ -12,7 +12,8 @@ class Client @response = send(orig_request_method, req, options) end - if HttpLog.url_approved?(req.uri) + uri = req.uri + if HttpLog.url_approved?(uri) body = if defined?(::HTTP::Request::Body) req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body) else @@ -21,7 +22,7 @@ class Client HttpLog.call( method: req.verb, - url: req.uri, + url: uri, request_body: body, request_headers: req.headers, response_code: @response.code, @@ -29,7 +30,8 @@ class Client response_headers: @response.headers, benchmark: bm, encoding: @response.headers['Content-Encoding'], - content_type: @response.headers['Content-Type'] + content_type: @response.headers['Content-Type'], + mask_body: HttpLog.masked_body_url?(uri) ) body.rewind if body.respond_to?(:rewind) diff --git a/lib/httplog/adapters/httpclient.rb b/lib/httplog/adapters/httpclient.rb index 89cbbe0..ea2a33e 100644 --- a/lib/httplog/adapters/httpclient.rb +++ b/lib/httplog/adapters/httpclient.rb @@ -16,12 +16,13 @@ def do_get_block(req, proxy, conn, &block) end end - if HttpLog.url_approved?(req.header.request_uri) + request_uri = req.header.request_uri + if HttpLog.url_approved?(request_uri) res = conn.pop HttpLog.call( method: req.header.request_method, - url: req.header.request_uri, + url: request_uri, request_body: req.body, request_headers: req.headers, response_code: res.status_code, @@ -29,7 +30,8 @@ def do_get_block(req, proxy, conn, &block) response_headers: res.headers, benchmark: bm, encoding: res.headers['Content-Encoding'], - content_type: res.headers['Content-Type'] + content_type: res.headers['Content-Type'], + mask_body: HttpLog.masked_body_url?(request_uri) ) conn.push(res) end diff --git a/lib/httplog/adapters/net_http.rb b/lib/httplog/adapters/net_http.rb index be19499..ac5efe3 100644 --- a/lib/httplog/adapters/net_http.rb +++ b/lib/httplog/adapters/net_http.rb @@ -23,7 +23,8 @@ def request(req, body = nil, &block) response_headers: @response.each_header.collect, benchmark: bm, encoding: @response['Content-Encoding'], - content_type: @response['Content-Type'] + content_type: @response['Content-Type'], + mask_body: HttpLog.masked_body_url?(url) ) end diff --git a/lib/httplog/adapters/patron.rb b/lib/httplog/adapters/patron.rb index 9716863..7dd5a8e 100644 --- a/lib/httplog/adapters/patron.rb +++ b/lib/httplog/adapters/patron.rb @@ -20,7 +20,8 @@ def request(action_name, url, headers, options = {}) response_headers: @response.headers, benchmark: bm, encoding: @response.headers['Content-Encoding'], - content_type: @response.headers['Content-Type'] + content_type: @response.headers['Content-Type'], + mask_body: HttpLog.masked_body_url?(url) ) end diff --git a/lib/httplog/configuration.rb b/lib/httplog/configuration.rb index b99d49d..2faf5d9 100644 --- a/lib/httplog/configuration.rb +++ b/lib/httplog/configuration.rb @@ -19,35 +19,39 @@ class Configuration :log_benchmark, :url_whitelist_pattern, :url_blacklist_pattern, + :url_masked_body_pattern, :color, :prefix_data_lines, :prefix_response_lines, :prefix_line_numbers, + :json_parser, :filter_parameters def initialize - @enabled = true - @compact_log = false - @json_log = false - @graylog = false - @logger = Logger.new($stdout) - @logger_method = :log - @severity = Logger::Severity::DEBUG - @prefix = LOG_PREFIX - @log_connect = true - @log_request = true - @log_headers = false - @log_data = true - @log_status = true - @log_response = true - @log_benchmark = true - @url_whitelist_pattern = nil - @url_blacklist_pattern = nil - @color = false - @prefix_data_lines = false - @prefix_response_lines = false - @prefix_line_numbers = false - @filter_parameters = [] + @enabled = true + @compact_log = false + @json_log = false + @graylog = false + @logger = Logger.new($stdout) + @logger_method = :log + @severity = Logger::Severity::DEBUG + @prefix = LOG_PREFIX + @log_connect = true + @log_request = true + @log_headers = false + @log_data = true + @log_status = true + @log_response = true + @log_benchmark = true + @url_whitelist_pattern = nil + @url_blacklist_pattern = nil + @url_masked_body_pattern = nil + @color = false + @prefix_data_lines = false + @prefix_response_lines = false + @prefix_line_numbers = false + @json_parser = JSON + @filter_parameters = [] end end end diff --git a/lib/httplog/http_log.rb b/lib/httplog/http_log.rb index 302365c..85c5c18 100644 --- a/lib/httplog/http_log.rb +++ b/lib/httplog/http_log.rb @@ -29,6 +29,7 @@ def configure end def call(options = {}) + parse_request(options) if config.json_log log_json(options) elsif config.graylog @@ -42,7 +43,7 @@ def call(options = {}) HttpLog.log_status(options[:response_code]) HttpLog.log_benchmark(options[:benchmark]) HttpLog.log_headers(options[:response_headers]) - HttpLog.log_body(options[:response_body], options[:encoding], options[:content_type]) + HttpLog.log_body(options[:response_body], options[:mask_body], options[:encoding], options[:content_type]) end end @@ -52,10 +53,14 @@ def url_approved?(url) !config.url_whitelist_pattern || url.to_s.match(config.url_whitelist_pattern) end + def masked_body_url?(url) + config.filter_parameters.any? && config.url_masked_body_pattern && url.to_s.match(config.url_masked_body_pattern) + end + def log(msg) return unless config.enabled - config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg)) + config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg.to_s)) end def log_connection(host, port = nil) @@ -91,10 +96,10 @@ def log_benchmark(seconds) log("Benchmark: #{seconds.to_f.round(6)} seconds") end - def log_body(body, encoding = nil, content_type = nil) + def log_body(body, mask_body, encoding = nil, content_type = nil) return unless config.log_response - data = parse_body(body, encoding, content_type) + data = parse_body(body.dup, mask_body, encoding, content_type) if config.prefix_response_lines log('Response:') @@ -106,7 +111,7 @@ def log_body(body, encoding = nil, content_type = nil) log("Response: #{e.message}") end - def parse_body(body, encoding, content_type) + def parse_body(body, mask_body, encoding, content_type) raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type) if body.is_a?(Net::ReadAdapter) @@ -125,14 +130,25 @@ def parse_body(body, encoding, content_type) end end - utf_encoded(body.to_s, content_type) + result = utf_encoded(body.to_s, content_type) + + if mask_body && body && !body.empty? + if content_type =~ /json/ + result = begin + masked_data config.json_parser.load(result) + rescue => e + 'Failed to mask response body: ' + e.message + end + else + result = masked(result) + end + end + result end def log_data(data) return unless config.log_data - data = utf_encoded(masked(data.dup).to_s) unless data.nil? - if config.prefix_data_lines log('Data:') log_data_lines(data) @@ -172,22 +188,43 @@ def colorize(msg) def log_json(data = {}) return unless config.json_log - log(json_payload(data).to_json) + log( + begin + dump_json(data) + rescue + data[:response_body] = "#{config.json_parser} dump failed" + data[:request_body] = "#{config.json_parser} dump failed" + dump_json(data) + end + ) + end + + def dump_json(data) + config.json_parser.dump(json_payload(data)) end - def log_graylog(data = {}) + def log_graylog(data) result = json_payload(data) - result[:rounded_benchmark] = data[:benchmark].round - result[:short_message] = result.delete(:url) - config.logger.public_send(config.logger_method, config.severity, result) + result[:short_message] = result.delete(:url) + begin + send_to_graylog result + rescue + result[:response_body] = 'Graylog JSON dump failed' + result[:request_body] = 'Graylog JSON dump failed' + send_to_graylog result + end + end + + def send_to_graylog data + config.logger.public_send(config.logger_method, config.severity, data) end def json_payload(data = {}) data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol) parsed_body = begin - parse_body(data[:response_body], data[:encoding], data[:content_type]) + parse_body(data[:response_body].dup, data[:mask_body], data[:encoding], data[:content_type]) rescue BodyParsingError => e e.message end @@ -203,7 +240,7 @@ def json_payload(data = {}) { method: data[:method].to_s.upcase, url: masked(data[:url]), - request_body: masked(data[:request_body]), + request_body: data[:request_body], request_headers: masked(data[:request_headers].to_h), response_code: data[:response_code].to_i, response_body: parsed_body, @@ -236,6 +273,38 @@ def masked(msg, key = nil) end end + def parse_request(options) + return if options[:request_body].nil? + + # Downcase content-type and content-encoding because ::HTTP returns "Content-Type" and "Content-Encoding" + headers = options[:request_headers].find_all do |header, _| + %w[content-type Content-Type content-encoding Content-Encoding].include? header + end.to_h.each_with_object({}) { |(k, v), h| h[k.downcase] = v } + + copy = options[:request_body].dup + + options[:request_body] = if text_based?(headers['content-type']) && options[:mask_body] + begin + parse_body(copy, options[:mask_body], headers['content-encoding'], headers['content-type']) + rescue BodyParsingError => e + log(e.message) + end + else + masked(copy).to_s + end + end + + def masked_data msg + case msg + when Hash + Hash[msg.map { |k, v| [k, config.filter_parameters.include?(k.downcase) ? PARAM_MASK : masked_data(v)] }] + when Array + msg.map { |element| masked_data(element) } + else + msg + end + end + def string_classes @string_classes ||= begin string_classes = [String] diff --git a/spec/adapters/net_http_adapter.rb b/spec/adapters/net_http_adapter.rb index aef2a88..be9061f 100644 --- a/spec/adapters/net_http_adapter.rb +++ b/spec/adapters/net_http_adapter.rb @@ -11,7 +11,7 @@ def send_head_request end def send_post_request - Net::HTTP.new(@host, @port).post(@path, @data) + Net::HTTP.new(@host, @port).post(@path, @data, @headers) end def send_post_form_request diff --git a/spec/lib/http_log_spec.rb b/spec/lib/http_log_spec.rb index 9c61641..12fc7ef 100644 --- a/spec/lib/http_log_spec.rb +++ b/spec/lib/http_log_spec.rb @@ -17,25 +17,27 @@ let(:gray_log) { JSON.parse("{#{log.match(/{(.*)/).captures.first}") } # Default configuration - let(:logger) { Logger.new @log } - let(:enabled) { HttpLog.configuration.enabled } - let(:severity) { HttpLog.configuration.severity } - let(:log_headers) { HttpLog.configuration.log_headers } - let(:log_request) { HttpLog.configuration.log_request } - let(:log_response) { HttpLog.configuration.log_response } - let(:log_data) { HttpLog.configuration.log_data } - let(:log_connect) { HttpLog.configuration.log_connect } - let(:log_benchmark) { HttpLog.configuration.log_benchmark } - let(:color) { HttpLog.configuration.color } - let(:prefix) { HttpLog.configuration.prefix } - let(:prefix_response_lines) { HttpLog.configuration.prefix_response_lines } - let(:prefix_line_numbers) { HttpLog.configuration.prefix_line_numbers } - let(:json_log) { HttpLog.configuration.json_log } - let(:graylog) { HttpLog.configuration.graylog } - let(:compact_log) { HttpLog.configuration.compact_log } - let(:url_blacklist_pattern) { HttpLog.configuration.url_blacklist_pattern } - let(:url_whitelist_pattern) { HttpLog.configuration.url_whitelist_pattern } - let(:filter_parameters) { HttpLog.configuration.filter_parameters } + let(:logger) { Logger.new @log } + let(:enabled) { HttpLog.configuration.enabled } + let(:severity) { HttpLog.configuration.severity } + let(:log_headers) { HttpLog.configuration.log_headers } + let(:log_request) { HttpLog.configuration.log_request } + let(:log_response) { HttpLog.configuration.log_response } + let(:log_data) { HttpLog.configuration.log_data } + let(:log_connect) { HttpLog.configuration.log_connect } + let(:log_benchmark) { HttpLog.configuration.log_benchmark } + let(:color) { HttpLog.configuration.color } + let(:prefix) { HttpLog.configuration.prefix } + let(:prefix_response_lines) { HttpLog.configuration.prefix_response_lines } + let(:prefix_line_numbers) { HttpLog.configuration.prefix_line_numbers } + let(:json_log) { HttpLog.configuration.json_log } + let(:graylog) { HttpLog.configuration.graylog } + let(:compact_log) { HttpLog.configuration.compact_log } + let(:url_blacklist_pattern) { HttpLog.configuration.url_blacklist_pattern } + let(:url_whitelist_pattern) { HttpLog.configuration.url_whitelist_pattern } + let(:json_parser) { HttpLog.configuration.json_parser } + let(:filter_parameters) { HttpLog.configuration.filter_parameters } + let(:url_masked_body_pattern) { HttpLog.configuration.url_masked_body_pattern } def configure HttpLog.configure do |c| @@ -57,7 +59,9 @@ def configure c.compact_log = compact_log c.url_blacklist_pattern = url_blacklist_pattern c.url_whitelist_pattern = url_whitelist_pattern + c.json_parser = json_parser c.filter_parameters = filter_parameters + c.url_masked_body_pattern = url_masked_body_pattern end end @@ -300,6 +304,88 @@ def configure it_behaves_like 'logs JSON', adapter_class, true end + + context 'with custom JSON parser' do + let(:json_log) { true } + let(:json_parser) { Oj } + + it_behaves_like 'logs JSON', adapter_class, false + end + + context 'with masked JSON and not JSON data' do + let(:url_masked_body_pattern) { /.*/ } + let(:params) { { 'foo' => secret } } + + # It shouldn't crash functionality with not JSON data + it_behaves_like 'filtered parameters' + end + + context 'with default JSON parser' do + let(:url_masked_body_pattern) { /.*/ } + it_behaves_like 'with masked JSON', adapter_class + end + + context 'pattern for masking JSON body' do + let(:url_masked_body_pattern) { /index/ } + it_behaves_like 'with masked JSON', adapter_class + end + + context ' URL not matches pattern for masking JSON body' do + let(:url_masked_body_pattern) { /not_matches/ } + let(:json_log) { true } + let(:path) { '/index.json' } + let(:headers) { { 'accept' => 'application/json', 'foo' => secret, 'content-type' => 'application/json' } } + let(:filter_parameters) { %w[foo] } + before { adapter.send_post_request } + + if adapter_class.method_defined? :send_post_request + it { expect(json['response_body'].to_s).to include(secret) } + end + end + + context 'with custom JSON parser' do + let(:url_masked_body_pattern) { /.*/ } + let(:json_parser) { Oj } + + it_behaves_like 'with masked JSON', adapter_class + end + + context 'masked with invalid JSON request' do + let(:json_log) { true } + let(:path) { '/index.json' } + let(:headers) { { 'accept' => 'application/json', 'foo' => secret, 'content-type' => 'application/json' } } + let(:url_masked_body_pattern) { /.*/ } + let(:data) do + '{foo:"my secret","bar":"baz","array":[{"foo":"my secret","bar":"baz"},{"hash":{"foo":"my secret","bar":"baz"}}]}' + end + let(:filter_parameters) { %w[foo] } + before { adapter.send_post_request } + + if adapter_class.method_defined? :send_post_request + it { expect(json['request_headers'].to_s).not_to include(secret) } + it { expect(json['request_body'].to_s).to include(secret) } + it { expect(json['response_body'].to_s).not_to include(secret) } + end + end + + describe 'Non UTF-8 with JSON log' do + if adapter_class.method_defined? :send_post_request + let!(:res) { adapter.send_post_request } + let(:json_log) { true } + + it { expect(res).to be_a adapter.response if adapter.respond_to? :response } + + context 'with non-UTF request data' do + let(:data) { "a UTF-8 striñg with an 8BIT-ASCII character: \xC3" } + it { is_expected.to include("request_body") } # == doesn't throw exception + end + + context 'with URI encoded non-UTF data' do + let(:data) { 'a UTF-8 striñg with a URI encoded 8BIT-ASCII character: %c3' } + it { is_expected.to include("request_body") } # == doesn't throw exception + end + end + end end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 0fb357a..c9d2f52 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -10,6 +10,7 @@ require 'patron' require 'http' require 'simplecov' +require 'oj' SimpleCov.start @@ -27,6 +28,8 @@ sleep(3) # wait a moment for the server to be booted RSpec.configure do |config| + Oj.default_options = {mode: :compat} + config.before(:each) do require 'stringio' diff --git a/spec/support/index.json b/spec/support/index.json new file mode 100644 index 0000000..b73a5a8 --- /dev/null +++ b/spec/support/index.json @@ -0,0 +1 @@ +{"foo":"my secret","bar":"baz","array":[{"foo":"my secret","bar":"baz"},{"hash":{"foo":"my secret","bar":"baz"}}]} \ No newline at end of file diff --git a/spec/support/shared_examples.rb b/spec/support/shared_examples.rb index 7c9fe70..d059933 100644 --- a/spec/support/shared_examples.rb +++ b/spec/support/shared_examples.rb @@ -91,7 +91,6 @@ it { expect(result['response_body']).to eq(html) } it { expect(result['benchmark']).to be_a(Numeric) } if gray - it { expect(result['rounded_benchmark']).to be_a(Integer) } it { expect(result['short_message']).to be_a(String) } end it_behaves_like 'filtered parameters' @@ -109,3 +108,25 @@ end end end + +RSpec.shared_examples 'with masked JSON' do |adapter_class| + if adapter_class.method_defined? :send_post_request + let(:json_log) { true } + let(:path) { '/index.json' } + let(:headers) { { 'accept' => 'application/json', 'foo' => secret, 'content-type' => 'application/json' } } + let(:data) do + '{"foo":"my secret","bar":"baz","array":[{"foo":"my secret","bar":"baz"},{"hash":{"foo":"my secret","bar":"baz"}}]}' + end + let(:filter_parameters) { %w[foo] } + before { adapter.send_post_request } + + it { expect(json['request_headers'].to_s).not_to include(secret) } + it { expect(json['request_body'].to_s).to include('hash') } + it { expect(json['request_body'].to_s).to include('[FILTERED]') } + it { expect(json['request_body'].to_s).not_to include(secret) } + + it { expect(json['response_body'].to_s).to include('hash') } + it { expect(json['response_body'].to_s).to include('[FILTERED]') } + it { expect(json['response_body'].to_s).not_to include(secret) } + end +end diff --git a/spec/support/test_server.rb b/spec/support/test_server.rb index 96e13cb..7a5c63b 100644 --- a/spec/support/test_server.rb +++ b/spec/support/test_server.rb @@ -23,6 +23,7 @@ def call(env) headers['Content-Type'] = 'application/octet-stream' if File.extname(file) == '.bin' headers['Content-Type'] = 'application/pdf' if File.extname(file) == '.pdf' headers['Content-Type'] = 'text/html; charset=UTF-8' if path =~ /utf8/ + headers['Content-Type'] = 'application/json' if path =~ /json/ headers['Content-Encoding'] = 'gzip' if File.extname(file) == '.gz' [200, headers, File.binread(file)] else