Allow deep mask json and custom JSON serializer. Fix non UTF-8 for js…

…on and graylog formats.

Gemfile.lock

@@ -70,6 +70,7 @@ GEM
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
+    oj (3.9.2)
     patron (0.13.3)
     pry (0.12.2)
       coderay (~> 1.1.0)
@@ -122,6 +123,7 @@ DEPENDENCIES
   httpclient (~> 2.8)
   httplog!
   listen (~> 3.0)
+  oj (>= 3.9.2)
   patron (~> 0.12)
   rake (~> 12.3)
   rspec (~> 3.7)

README.md

@@ -86,7 +86,14 @@ HttpLog.configure do |config|
   config.url_whitelist_pattern = nil
   config.url_blacklist_pattern = nil
 
-  # Mask the values of sensitive requestparameters
+  # Mask sensitive information in request and response JSON data.
+  # Enable global JSON masking by setting the parameter to `/.*/`
+  config.url_masked_body_pattern = nil
+
+  # You can specify any custom JSON serializer that implements `load` and `dump` class methods
+  config.json_parser = JSON
+
+  # Mask the values of sensitive request parameters
   config.filter_parameters = %w[password]
 end
 ```
@@ -121,7 +128,7 @@ HttpLog.configure do |config|
 end
 ```
 
-If you use Graylog and want to use its search features such as "rounded_benchmark:>1 AND method:PUT",
+If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT",
 you can use this configuration:
 
 ```ruby

httplog.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rspec', ['~> 3.7']
   gem.add_development_dependency 'simplecov', ['~> 0.15']
   gem.add_development_dependency 'thin', ['~> 1.7']
+  gem.add_development_dependency 'oj', ['>= 3.9.2']
 
   gem.add_dependency 'rack', ['>= 1.0']
   gem.add_dependency 'rainbow', ['>= 2.0.0']

lib/httplog/adapters/ethon.rb

@@ -39,7 +39,8 @@ def perform
             response_headers: headers.map { |header| header.split(/:\s/) }.to_h,
             benchmark: bm,
             encoding: encoding,
-            content_type: content_type
+            content_type: content_type,
+            mask_body: HttpLog.masked_body_url?(url)
           )
           return_code
         end

lib/httplog/adapters/excon.rb

@@ -45,7 +45,8 @@ def request(params, &block)
           response_headers: headers,
           benchmark: bm,
           encoding: headers['Content-Encoding'],
-          content_type: headers['Content-Type']
+          content_type: headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
         result
       end

lib/httplog/adapters/http.rb

@@ -12,7 +12,8 @@ class Client
           @response = send(orig_request_method, req, options)
         end
 
-        if HttpLog.url_approved?(req.uri)
+        uri = req.uri
+        if HttpLog.url_approved?(uri)
           body = if defined?(::HTTP::Request::Body)
                    req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body)
                  else
@@ -21,15 +22,16 @@ class Client
 
           HttpLog.call(
             method: req.verb,
-            url: req.uri,
+            url: uri,
             request_body: body,
             request_headers: req.headers,
             response_code: @response.code,
             response_body: @response.body,
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(uri)
           )
 
           body.rewind if body.respond_to?(:rewind)

lib/httplog/adapters/httpclient.rb

@@ -16,20 +16,22 @@ def do_get_block(req, proxy, conn, &block)
         end
       end
 
-      if HttpLog.url_approved?(req.header.request_uri)
+      request_uri = req.header.request_uri
+      if HttpLog.url_approved?(request_uri)
         res = conn.pop
 
         HttpLog.call(
           method: req.header.request_method,
-          url: req.header.request_uri,
+          url: request_uri,
           request_body: req.body,
           request_headers: req.headers,
           response_code: res.status_code,
           response_body: res.body,
           response_headers: res.headers,
           benchmark: bm,
           encoding: res.headers['Content-Encoding'],
-          content_type: res.headers['Content-Type']
+          content_type: res.headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(request_uri)
         )
         conn.push(res)
       end

lib/httplog/adapters/net_http.rb

@@ -23,7 +23,8 @@ def request(req, body = nil, &block)
           response_headers: @response.each_header.collect,
           benchmark: bm,
           encoding: @response['Content-Encoding'],
-          content_type: @response['Content-Type']
+          content_type: @response['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
       end
 

lib/httplog/adapters/patron.rb

@@ -20,7 +20,8 @@ def request(action_name, url, headers, options = {})
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(url)
           )
         end
 

lib/httplog/configuration.rb

@@ -19,35 +19,39 @@ class Configuration
                   :log_benchmark,
                   :url_whitelist_pattern,
                   :url_blacklist_pattern,
+                  :url_masked_body_pattern,
                   :color,
                   :prefix_data_lines,
                   :prefix_response_lines,
                   :prefix_line_numbers,
+                  :json_parser,
                   :filter_parameters
 
     def initialize
-      @enabled               = true
-      @compact_log           = false
-      @json_log              = false
-      @graylog               = false
-      @logger                = Logger.new($stdout)
-      @logger_method         = :log
-      @severity              = Logger::Severity::DEBUG
-      @prefix                = LOG_PREFIX
-      @log_connect           = true
-      @log_request           = true
-      @log_headers           = false
-      @log_data              = true
-      @log_status            = true
-      @log_response          = true
-      @log_benchmark         = true
-      @url_whitelist_pattern = nil
-      @url_blacklist_pattern = nil
-      @color                 = false
-      @prefix_data_lines     = false
-      @prefix_response_lines = false
-      @prefix_line_numbers   = false
-      @filter_parameters     = []
+      @enabled                 = true
+      @compact_log             = false
+      @json_log                = false
+      @graylog                 = false
+      @logger                  = Logger.new($stdout)
+      @logger_method           = :log
+      @severity                = Logger::Severity::DEBUG
+      @prefix                  = LOG_PREFIX
+      @log_connect             = true
+      @log_request             = true
+      @log_headers             = false
+      @log_data                = true
+      @log_status              = true
+      @log_response            = true
+      @log_benchmark           = true
+      @url_whitelist_pattern   = nil
+      @url_blacklist_pattern   = nil
+      @url_masked_body_pattern = nil
+      @color                   = false
+      @prefix_data_lines       = false
+      @prefix_response_lines   = false
+      @prefix_line_numbers     = false
+      @json_parser             = JSON
+      @filter_parameters       = []
     end
   end
 end

lib/httplog/http_log.rb

@@ -29,6 +29,7 @@ def configure
     end
 
     def call(options = {})
+      parse_request(options)
       if config.json_log
         log_json(options)
       elsif config.graylog
@@ -42,7 +43,7 @@ def call(options = {})
         HttpLog.log_status(options[:response_code])
         HttpLog.log_benchmark(options[:benchmark])
         HttpLog.log_headers(options[:response_headers])
-        HttpLog.log_body(options[:response_body], options[:encoding], options[:content_type])
+        HttpLog.log_body(options[:response_body], options[:mask_body], options[:encoding], options[:content_type])
       end
     end
 
@@ -52,10 +53,14 @@ def url_approved?(url)
       !config.url_whitelist_pattern || url.to_s.match(config.url_whitelist_pattern)
     end
 
+    def masked_body_url?(url)
+      config.filter_parameters.any? && config.url_masked_body_pattern && url.to_s.match(config.url_masked_body_pattern)
+    end
+
     def log(msg)
       return unless config.enabled
 
-      config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg))
+      config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg.to_s))
     end
 
     def log_connection(host, port = nil)
@@ -91,10 +96,10 @@ def log_benchmark(seconds)
       log("Benchmark: #{seconds.to_f.round(6)} seconds")
     end
 
-    def log_body(body, encoding = nil, content_type = nil)
+    def log_body(body, mask_body, encoding = nil, content_type = nil)
       return unless config.log_response
 
-      data = parse_body(body, encoding, content_type)
+      data = parse_body(body.dup, mask_body, encoding, content_type)
 
       if config.prefix_response_lines
         log('Response:')
@@ -106,7 +111,7 @@ def log_body(body, encoding = nil, content_type = nil)
       log("Response: #{e.message}")
     end
 
-    def parse_body(body, encoding, content_type)
+    def parse_body(body, mask_body, encoding, content_type)
       raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type)
 
       if body.is_a?(Net::ReadAdapter)
@@ -125,14 +130,25 @@ def parse_body(body, encoding, content_type)
         end
       end
 
-      utf_encoded(body.to_s, content_type)
+      result = utf_encoded(body.to_s, content_type)
+
+      if mask_body && body && !body.empty?
+        if content_type =~ /json/
+          result = begin
+                     masked_data config.json_parser.load(result)
+                   rescue => e
+                     'Failed to mask response body: ' + e.message
+                   end
+        else
+          result = masked(result)
+        end
+      end
+      result
     end
 
     def log_data(data)
       return unless config.log_data
 
-      data = utf_encoded(masked(data.dup).to_s) unless data.nil?
-
       if config.prefix_data_lines
         log('Data:')
         log_data_lines(data)
@@ -172,22 +188,43 @@ def colorize(msg)
     def log_json(data = {})
       return unless config.json_log
 
-      log(json_payload(data).to_json)
+      log(
+        begin
+          dump_json(data)
+        rescue
+          data[:response_body] = "#{config.json_parser} dump failed"
+          data[:request_body]  = "#{config.json_parser} dump failed"
+          dump_json(data)
+        end
+      )
+    end
+
+    def dump_json(data)
+      config.json_parser.dump(json_payload(data))
     end
 
-    def log_graylog(data = {})
+    def log_graylog(data)
       result = json_payload(data)
 
-      result[:rounded_benchmark] = data[:benchmark].round
-      result[:short_message]     = result.delete(:url)
-      config.logger.public_send(config.logger_method, config.severity, result)
+      result[:short_message] = result.delete(:url)
+      begin
+        send_to_graylog result
+      rescue
+        result[:response_body] = 'Graylog JSON dump failed'
+        result[:request_body]  = 'Graylog JSON dump failed'
+        send_to_graylog result
+      end
+    end
+
+    def send_to_graylog data
+      config.logger.public_send(config.logger_method, config.severity, data)
     end
 
     def json_payload(data = {})
       data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol)
 
       parsed_body = begin
-                      parse_body(data[:response_body], data[:encoding], data[:content_type])
+                      parse_body(data[:response_body].dup, data[:mask_body], data[:encoding], data[:content_type])
                     rescue BodyParsingError => e
                       e.message
                     end
@@ -203,7 +240,7 @@ def json_payload(data = {})
         {
           method: data[:method].to_s.upcase,
           url: masked(data[:url]),
-          request_body: masked(data[:request_body]),
+          request_body: data[:request_body],
           request_headers: masked(data[:request_headers].to_h),
           response_code: data[:response_code].to_i,
           response_body: parsed_body,
@@ -236,6 +273,38 @@ def masked(msg, key = nil)
       end
     end
 
+    def parse_request(options)
+      return if options[:request_body].nil?
+
+      # Downcase content-type and content-encoding because ::HTTP returns "Content-Type" and "Content-Encoding"
+      headers = options[:request_headers].find_all do |header, _|
+        %w[content-type Content-Type content-encoding Content-Encoding].include? header
+      end.to_h.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
+
+      copy = options[:request_body].dup
+
+      options[:request_body] = if text_based?(headers['content-type']) && options[:mask_body]
+                                 begin
+                                   parse_body(copy, options[:mask_body], headers['content-encoding'], headers['content-type'])
+                                 rescue BodyParsingError => e
+                                   log(e.message)
+                                 end
+                               else
+                                 masked(copy).to_s
+                               end
+    end
+
+    def masked_data msg
+      case msg
+      when Hash
+        Hash[msg.map { |k, v| [k, config.filter_parameters.include?(k.downcase) ? PARAM_MASK : masked_data(v)] }]
+      when Array
+        msg.map { |element| masked_data(element) }
+      else
+        msg
+      end
+    end
+
     def string_classes
       @string_classes ||= begin
         string_classes = [String]

spec/adapters/net_http_adapter.rb

@@ -11,7 +11,7 @@ def send_head_request
   end
 
   def send_post_request
-    Net::HTTP.new(@host, @port).post(@path, @data)
+    Net::HTTP.new(@host, @port).post(@path, @data, @headers)
   end
 
   def send_post_form_request