Set response String encoding

Parse content-type response header with some helps from OpenURI::Meta
and set response String encoding.  Closes #26.

lib/httpclient.rb

@@ -1062,7 +1062,7 @@ def do_get_block(req, proxy, conn, &block)
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
-      force_binary(part)
+      set_encoding(part, res.body_encoding)
       if block
         block.call(res, part)
       else
@@ -1098,7 +1098,7 @@ def do_get_stream(req, proxy, conn)
     do_get_header(req, res, sess)
     conn.push(res)
     sess.get_body do |part|
-      force_binary(part)
+      set_encoding(part, res.body_encoding)
       pipew.write(part)
     end
     pipew.close
@@ -1111,9 +1111,7 @@ def do_get_stream(req, proxy, conn)
 
   def do_get_header(req, res, sess)
     res.http_version, res.status, res.reason, headers = sess.get_header
-    headers.each do |key, value|
-      res.header.add(key, value)
-    end
+    res.header.set_headers(headers)
     if @cookie_manager
       res.header['set-cookie'].each do |cookie|
         @cookie_manager.parse(cookie, req.header.request_uri)
@@ -1127,4 +1125,8 @@ def dump_dummy_request_response(req, res)
     @debug_dev << "\n\n= Dummy Response\n\n"
     @debug_dev << res
   end
+
+  def set_encoding(str, encoding)
+    str.force_encoding(encoding) if encoding
+  end
 end

lib/httpclient/http.rb

@@ -7,6 +7,7 @@
 
 
 require 'time'
+require 'open-uri' # for encoding
 
 
 # A namespace module for HTTP Message definitions used by HTTPClient.
@@ -122,6 +123,8 @@ class Headers
       attr_accessor :body_charset # :nodoc:
       # Used for dumping response.
       attr_accessor :body_date # :nodoc:
+      # Used for keeping content encoding.
+      attr_reader :body_encoding # :nodoc:
 
       # HTTP response status code to reason phrase mapping definition.
       STATUS_CODE_MAP = {
@@ -166,6 +169,7 @@ def initialize
         @body_type = nil
         @body_charset = nil
         @body_date = nil
+        @body_encoding = nil
 
         @is_request = nil
         @header_item = []
@@ -205,14 +209,31 @@ def status_code=(status_code)
       end
 
       # Returns 'Content-Type' header value.
-      def contenttype
+      def content_type
         self['Content-Type'][0]
       end
 
       # Sets 'Content-Type' header value.  Overrides if already exists.
-      def contenttype=(contenttype)
+      def content_type=(content_type)
         delete('Content-Type')
-        self['Content-Type'] = contenttype
+        self['Content-Type'] = content_type
+      end
+
+      alias contenttype content_type
+      alias contenttype= content_type=
+
+      if defined?(Encoding::ASCII_8BIT)
+        def set_body_encoding
+          if type = self.content_type
+            OpenURI::Meta.init(o = '')
+            o.meta_add_field('content-type', type)
+            @body_encoding = o.encoding
+          end
+        end
+      else
+        def set_body_encoding
+          @body_encoding = nil
+        end
       end
 
       # Sets byte size of message body.
@@ -285,6 +306,13 @@ def [](key)
         get(key).collect { |item| item[1] }
       end
 
+      def set_headers(headers)
+        headers.each do |key, value|
+          add(key, value)
+        end
+        set_body_encoding
+      end
+
       def create_query_uri()
         if @request_method == 'CONNECT'
           return "#{@request_uri.host}:#{@request_uri.port}"
@@ -917,14 +945,21 @@ def reason=(reason)
       @http_header.reason_phrase = reason
     end
 
+    # Returns 'Content-Type' header value.
+    def content_type
+      @http_header.content_type
+    end
+
     # Sets 'Content-Type' header value.  Overrides if already exists.
-    def contenttype
-      @http_header.contenttype
+    def content_type=(content_type)
+      @http_header.content_type = content_type
     end
+    alias contenttype content_type
+    alias contenttype= content_type=
 
-    # Returns 'Content-Type' header value.
-    def contenttype=(contenttype)
-      @http_header.contenttype = contenttype
+    # Returns content encoding
+    def body_encoding
+      @http_header.body_encoding
     end
 
     # Returns a content of message body.  A String or an IO.

lib/httpclient/util.rb

@@ -116,17 +116,6 @@ def hash_find_value(hash, &block)
     def https?(uri)
       uri.scheme.downcase == 'https'
     end
-
-    if defined?(Encoding::ASCII_8BIT)
-      def force_binary(str)
-        str.force_encoding(Encoding::ASCII_8BIT)
-        str
-      end
-    else
-      def force_binary(str)
-        str
-      end
-    end
   end
 
 

test/test_httpclient.rb

@@ -1361,6 +1361,14 @@ def test_body_param_order
     assert_equal("key2=b&key2=c&key2=d&key1=a&key3=z", HTTP::Message.escape_query(ary))
   end
 
+  if RUBY_VERSION > "1.9"
+    def test_charset
+      body = @client.get(serverurl + 'charset').body
+      assert_equal(Encoding::EUC_JP, body.encoding)
+      assert_equal('あいうえお'.encode(Encoding::EUC_JP), body)
+    end
+  end
+
 private
 
   def check_query_get(query)
@@ -1384,7 +1392,7 @@ def setup_server
       :DocumentRoot => File.dirname(File.expand_path(__FILE__))
     )
     @serverport = @server.config[:Port]
-    [:hello, :sleep, :servlet_redirect, :redirect1, :redirect2, :redirect3, :redirect_self, :relative_redirect, :chunked, :largebody, :status, :compressed].each do |sym|
+    [:hello, :sleep, :servlet_redirect, :redirect1, :redirect2, :redirect3, :redirect_self, :relative_redirect, :chunked, :largebody, :status, :compressed, :charset].each do |sym|
       @server.mount(
 	"/#{sym}",
 	WEBrick::HTTPServlet::ProcHandler.new(method("do_#{sym}").to_proc)
@@ -1461,6 +1469,7 @@ def do_largebody(req, res)
   end
 
   def do_compressed(req, res)
+    res['content-type'] = 'application/octet-stream'
     if req.query['enc'] == 'gzip'
       res['content-encoding'] = 'gzip'
       res.body = GZIP_CONTENT
@@ -1470,6 +1479,15 @@ def do_compressed(req, res)
     end
   end
 
+  def do_charset(req, res)
+    if RUBY_VERSION > "1.9"
+      res.body = 'あいうえお'.encode("euc-jp")
+      res['Content-Type'] = 'text/plain; charset=euc-jp'
+    else
+      res.body = 'this endpoint is for 1.9 or later'
+    end
+  end
+
   def do_status(req, res)
     res.status = req.query['status'].to_i
   end
@@ -1490,6 +1508,7 @@ def do_GET(req, res)
     end
 
     def do_POST(req, res)
+      res["content-type"] = "text/plain" # iso-8859-1, not US-ASCII
       res.body = 'post,' + req.body.to_s
       res["x-query"] = body_response(req)
     end