Allow deep mask json and custom JSON serializer. Fix non UTF-8 for js…

.rubocop.yml

@@ -1,7 +1,7 @@
 inherit_from: ./.rubocop_todo.yml
 
 AllCops:
-  TargetRubyVersion: 2.2
+  TargetRubyVersion: 2.4
   Exclude:
     - 'db/**/*'
     - 'db/schema.rb'
@@ -28,7 +28,7 @@ Layout/EmptyLinesAroundClassBody:
 Layout/EmptyLinesAroundModuleBody:
   EnforcedStyle: no_empty_lines
 
-Layout/IndentArray:
+Layout/FirstArrayElementIndentation:
   EnforcedStyle: consistent
 
 Style/Documentation:

.rubocop_todo.yml

@@ -10,7 +10,7 @@ Metrics/BlockLength:
 # Offense count: 137
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
 # URISchemes: http, https
-Metrics/LineLength:
+Layout/LineLength:
   Enabled: false
 
 # Offense count: 7

.ruby-version

@@ -1 +1 @@
-2.6.3
+2.7.0

.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 rvm:
-  - "2.4.5"
-  - "2.5.3"
-  - "2.6.2"
+  - "2.4.9"
+  - "2.5.7"
+  - "2.6.5"
+  - "2.7.0"
 gemfile:
   - gemfiles/http3.gemfile
   - gemfiles/http4.gemfile

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.4.0 - TBD
+
+* Bumped ruby version to 2.7
+* Fixed ruby 2.7 deprecations
+* Removed support for ruby < 2.4
+
 ## 1.3.3 - 2019-11-14
 
 * [#83](https://github.com/trusche/httplog/pull/83) Support for graylog

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    httplog (1.3.3)
+    httplog (1.4.0)
       rack (>= 1.0)
       rainbow (>= 2.0.0)
 
@@ -70,6 +70,7 @@ GEM
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
+    oj (3.9.2)
     patron (0.13.3)
     pry (0.12.2)
       coderay (~> 1.1.0)
@@ -122,11 +123,12 @@ DEPENDENCIES
   httpclient (~> 2.8)
   httplog!
   listen (~> 3.0)
+  oj (>= 3.9.2)
   patron (~> 0.12)
   rake (~> 12.3)
   rspec (~> 3.7)
   simplecov (~> 0.15)
   thin (~> 1.7)
 
 BUNDLED WITH
-   2.0.2
+   2.1.2

README.md

@@ -86,7 +86,14 @@ HttpLog.configure do |config|
   config.url_whitelist_pattern = nil
   config.url_blacklist_pattern = nil
 
-  # Mask the values of sensitive requestparameters
+  # Mask sensitive information in request and response JSON data.
+  # Enable global JSON masking by setting the parameter to `/.*/`
+  config.url_masked_body_pattern = nil
+
+  # You can specify any custom JSON serializer that implements `load` and `dump` class methods
+  config.json_parser = JSON
+
+  # Mask the values of sensitive request parameters
   config.filter_parameters = %w[password]
 end
 ```
@@ -121,7 +128,7 @@ HttpLog.configure do |config|
 end
 ```
 
-If you use Graylog and want to use its search features such as "rounded_benchmark:>1 AND method:PUT",
+If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT",
 you can use this configuration:
 
 ```ruby

httplog.gemspec

@@ -18,11 +18,11 @@ Gem::Specification.new do |gem|
                      of third party gems that don't provide their own log output."
 
   gem.files         = Dir['lib/**/*.rb'] +
-                        %w(httplog.gemspec README.md CHANGELOG.md)
+                      %w[httplog.gemspec README.md CHANGELOG.md]
   gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.require_paths = ['lib']
 
-  gem.required_ruby_version = '>= 2.2'
+  gem.required_ruby_version = '>= 2.4'
 
   gem.add_development_dependency 'ethon', ['~> 0.11']
   gem.add_development_dependency 'excon', ['~> 0.60']
@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rspec', ['~> 3.7']
   gem.add_development_dependency 'simplecov', ['~> 0.15']
   gem.add_development_dependency 'thin', ['~> 1.7']
+  gem.add_development_dependency 'oj', ['>= 3.9.2']
 
   gem.add_dependency 'rack', ['>= 1.0']
   gem.add_dependency 'rainbow', ['>= 2.0.0']

lib/httplog/adapters/ethon.rb

@@ -39,7 +39,8 @@ def perform
             response_headers: headers.map { |header| header.split(/:\s/) }.to_h,
             benchmark: bm,
             encoding: encoding,
-            content_type: content_type
+            content_type: content_type,
+            mask_body: HttpLog.masked_body_url?(url)
           )
           return_code
         end

lib/httplog/adapters/excon.rb

@@ -45,7 +45,8 @@ def request(params, &block)
           response_headers: headers,
           benchmark: bm,
           encoding: headers['Content-Encoding'],
-          content_type: headers['Content-Type']
+          content_type: headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
         result
       end

lib/httplog/adapters/http.rb

@@ -12,7 +12,8 @@ class Client
           @response = send(orig_request_method, req, options)
         end
 
-        if HttpLog.url_approved?(req.uri)
+        uri = req.uri
+        if HttpLog.url_approved?(uri)
           body = if defined?(::HTTP::Request::Body)
                    req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body)
                  else
@@ -21,15 +22,16 @@ class Client
 
           HttpLog.call(
             method: req.verb,
-            url: req.uri,
+            url: uri,
             request_body: body,
             request_headers: req.headers,
             response_code: @response.code,
             response_body: @response.body,
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(uri)
           )
 
           body.rewind if body.respond_to?(:rewind)

lib/httplog/adapters/httpclient.rb

@@ -16,20 +16,22 @@ def do_get_block(req, proxy, conn, &block)
         end
       end
 
-      if HttpLog.url_approved?(req.header.request_uri)
+      request_uri = req.header.request_uri
+      if HttpLog.url_approved?(request_uri)
         res = conn.pop
 
         HttpLog.call(
           method: req.header.request_method,
-          url: req.header.request_uri,
+          url: request_uri,
           request_body: req.body,
           request_headers: req.headers,
           response_code: res.status_code,
           response_body: res.body,
           response_headers: res.headers,
           benchmark: bm,
           encoding: res.headers['Content-Encoding'],
-          content_type: res.headers['Content-Type']
+          content_type: res.headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(request_uri)
         )
         conn.push(res)
       end
@@ -44,17 +46,13 @@ class Session
       # it's `create_socket(host, port)`
       if instance_method(:create_socket).arity == 1
         def create_socket(site)
-          if HttpLog.url_approved?("#{site.host}:#{site.port}")
-            HttpLog.log_connection(site.host, site.port)
-          end
+          HttpLog.log_connection(site.host, site.port) if HttpLog.url_approved?("#{site.host}:#{site.port}")
           orig_create_socket(site)
         end
 
       else
         def create_socket(host, port)
-          if HttpLog.url_approved?("#{host}:#{port}")
-            HttpLog.log_connection(host, port)
-          end
+          HttpLog.log_connection(host, port) if HttpLog.url_approved?("#{host}:#{port}")
           orig_create_socket(host, port)
         end
       end

lib/httplog/adapters/net_http.rb

@@ -23,7 +23,8 @@ def request(req, body = nil, &block)
           response_headers: @response.each_header.collect,
           benchmark: bm,
           encoding: @response['Content-Encoding'],
-          content_type: @response['Content-Type']
+          content_type: @response['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
       end
 

lib/httplog/adapters/patron.rb

@@ -20,7 +20,8 @@ def request(action_name, url, headers, options = {})
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(url)
           )
         end
 

lib/httplog/configuration.rb

@@ -19,35 +19,39 @@ class Configuration
                   :log_benchmark,
                   :url_whitelist_pattern,
                   :url_blacklist_pattern,
+                  :url_masked_body_pattern,
                   :color,
                   :prefix_data_lines,
                   :prefix_response_lines,
                   :prefix_line_numbers,
+                  :json_parser,
                   :filter_parameters
 
     def initialize
-      @enabled               = true
-      @compact_log           = false
-      @json_log              = false
-      @graylog               = false
-      @logger                = Logger.new($stdout)
-      @logger_method         = :log
-      @severity              = Logger::Severity::DEBUG
-      @prefix                = LOG_PREFIX
-      @log_connect           = true
-      @log_request           = true
-      @log_headers           = false
-      @log_data              = true
-      @log_status            = true
-      @log_response          = true
-      @log_benchmark         = true
-      @url_whitelist_pattern = nil
-      @url_blacklist_pattern = nil
-      @color                 = false
-      @prefix_data_lines     = false
-      @prefix_response_lines = false
-      @prefix_line_numbers   = false
-      @filter_parameters     = []
+      @enabled                 = true
+      @compact_log             = false
+      @json_log                = false
+      @graylog                 = false
+      @logger                  = Logger.new($stdout)
+      @logger_method           = :log
+      @severity                = Logger::Severity::DEBUG
+      @prefix                  = LOG_PREFIX
+      @log_connect             = true
+      @log_request             = true
+      @log_headers             = false
+      @log_data                = true
+      @log_status              = true
+      @log_response            = true
+      @log_benchmark           = true
+      @url_whitelist_pattern   = nil
+      @url_blacklist_pattern   = nil
+      @url_masked_body_pattern = nil
+      @color                   = false
+      @prefix_data_lines       = false
+      @prefix_response_lines   = false
+      @prefix_line_numbers     = false
+      @json_parser             = JSON
+      @filter_parameters       = []
     end
   end
 end