Merge pull request #31 from nbulaj/big-refactoring

Big refactoring
nbulaj · Oct 23, 2019 · 061744b · 061744b
2 parents da2e067 + 29ad097
commit 061744b
Show file tree

Hide file tree

Showing 47 changed files with 451 additions and 419 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -1,14 +1,30 @@
-LineLength:
-  Max: 120
 AllCops:
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 2.3
   Exclude:
-    - 'spec/**/*'
     - 'bin/*'
   DisplayCopNames: true
-Rails:
-  Enabled: false
-Documentation:
-  Enabled: false
-FrozenStringLiteralComment:
-  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Exclude:
+    - spec/**/*
+Style/FrozenStringLiteralComment:
+  Enabled: true
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Layout/TrailingBlankLines:
+  Enabled: true
+Layout/DotPosition:
+  EnforcedStyle: leading
+
+Metrics/LineLength:
+  Exclude:
+    - spec/**/*
+  Max: 100
+Metrics/BlockLength:
+  Exclude:
+    - spec/**/*
diff --git a/.travis.yml b/.travis.yml
@@ -14,21 +14,15 @@ gemfile:
   - gemfiles/nokogiri.gemfile
 
 rvm:
-  - 2.0
-  - 2.1
-  - 2.2
   - 2.3
   - 2.4
   - 2.5
   - 2.6
   - ruby-head
-  - jruby-9.2.1
+  - jruby-9.2.8.0
 
 matrix:
   allow_failures:
     - rvm: ruby-head
-    - rvm: jruby-9.2.1
+    - rvm: jruby-9.2.8.0
     - rvm: truffleruby
-  exclude:
-    - rvm: 2.0
-      gemfile: gemfiles/nokogiri.gemfile # Nokogiri doesn't support Ruby 2.0
diff --git a/Gemfile b/Gemfile
@@ -1,11 +1,14 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
+
+source "https://rubygems.org"
 
 gemspec
 
-gem 'nokogiri', '~> 1.8'
-gem 'oga', '~> 2.0'
+gem "nokogiri", "~> 1.8"
+gem "oga", "~> 2.0"
+gem "rubocop", "~> 0.74"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
 end
diff --git a/Rakefile b/Rakefile
@@ -1,6 +1,8 @@
-require 'bundler/gem_tasks'
+# frozen_string_literal: true
 
-require 'rspec/core/rake_task'
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
diff --git a/gemfiles/nokogiri.gemfile b/gemfiles/nokogiri.gemfile
@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'nokogiri', '~> 1.8'
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.8"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.6"
 end
diff --git a/gemfiles/oga.gemfile b/gemfiles/oga.gemfile
@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'oga', '~> 2.0'
+gemspec path: "../"
+
+gem "oga", "~> 2.0"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.6"
 end
diff --git a/lib/proxy_fetcher.rb b/lib/proxy_fetcher.rb
@@ -1,44 +1,44 @@
 # frozen_string_literal: true
 
-require 'uri'
-require 'http'
-require 'logger'
+require "uri"
+require "http"
+require "logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/version'
+require File.dirname(__FILE__) + "/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/exceptions'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration/providers_registry'
-require File.dirname(__FILE__) + '/proxy_fetcher/proxy'
-require File.dirname(__FILE__) + '/proxy_fetcher/manager'
-require File.dirname(__FILE__) + '/proxy_fetcher/null_logger'
+require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
+require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
+require File.dirname(__FILE__) + "/proxy_fetcher/manager"
+require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/http_client'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_list_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/client'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/request'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/proxies_registry'
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/document'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/node'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/abstract_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/nokogiri_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/oga_adapter'
+require File.dirname(__FILE__) + "/proxy_fetcher/document"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/base'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list_ssl'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/gather_proxy'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/http_tunnel'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/xroxy'
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
   end
 
   # Main ProxyFetcher module.
@@ -75,7 +75,7 @@ def configure
 
     # Returns ProxyFetcher logger instance.
     #
-    # @return [Logger, NullLogger] logger object
+    # @return [Logger, ProxyFetcher::NullLogger] logger object
     #
     def logger
       return @logger if defined?(@logger)

diff --git a/lib/proxy_fetcher/client/client.rb b/lib/proxy_fetcher/client/client.rb
@@ -128,7 +128,11 @@ def patch(url, payload, headers: {}, options: {})
       #
       def request_with_payload(method, url, payload, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(payload: payload, proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            payload: payload,
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -138,7 +142,10 @@ def request_with_payload(method, url, payload, headers, options)
       #
       def request_without_payload(method, url, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -152,7 +159,7 @@ def request_without_payload(method, url, headers, options)
       #
       def default_headers
         {
-          'User-Agent' => ProxyFetcher.config.user_agent
+          "User-Agent" => ProxyFetcher.config.user_agent
         }
       end
 

diff --git a/lib/proxy_fetcher/client/request.rb b/lib/proxy_fetcher/client/request.rb
@@ -50,7 +50,7 @@ def self.execute(args)
       # @return [Request]
       #
       def initialize(args)
-        raise ArgumentError, 'args must be a Hash!' unless args.is_a?(Hash)
+        raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)
         @method = args.fetch(:method).to_s.downcase
@@ -86,9 +86,9 @@ def execute
       #
       def build_http_client
         HTTP.via(proxy.addr, proxy.port.to_i)
-            .headers(headers)
-            .timeout(connect: timeout, read: timeout)
-            .follow(max_hops: max_redirects)
+          .headers(headers)
+          .timeout(connect: timeout, read: timeout)
+          .follow(max_hops: max_redirects)
       end
 
       # Default SSL options that will be used for connecting to resources

diff --git a/lib/proxy_fetcher/configuration.rb b/lib/proxy_fetcher/configuration.rb
@@ -6,15 +6,20 @@ module ProxyFetcher
   #
   class Configuration
     # @!attribute client_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for [ProxyFetcher::Client]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for [ProxyFetcher::Client]
     attr_accessor :client_timeout
 
     # @!attribute provider_proxies_load_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for loading of proxies list by provider
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for loading
+    #     of proxies list by provider
     attr_accessor :provider_proxies_load_timeout
 
     # @!attribute proxy_validation_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for proxy
+    #     validation with [ProxyFetcher::ProxyValidator]
     attr_accessor :proxy_validation_timeout
 
     # to save compatibility
@@ -30,16 +35,12 @@ class Configuration
     attr_accessor :user_agent
 
     # @!attribute [r] logger
-    #   @return [Object] Logger object
+    #   @return [Logger] Logger object
     attr_accessor :logger
 
     # @!attribute [r] adapter
     #   @return [Object] HTML parser adapter
-    attr_accessor :adapter
-
-    # @!attribute [r] adapter_class
-    #   @return [Object] HTML adapter class
-    attr_reader :adapter_class
+    attr_reader :adapter
 
     # @!attribute [r] http_client
     #   @return [Object] HTTP client class
@@ -58,8 +59,8 @@ class Configuration
     #
     # Default is Google Chrome 60, but can be changed in <code>ProxyFetcher.config</code>.
     #
-    DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 ' \
-                         '(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36'.freeze
+    DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 " \
+                         "(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36"
 
     # HTML parser adapter name.
     #

diff --git a/lib/proxy_fetcher/document.rb b/lib/proxy_fetcher/document.rb
@@ -36,14 +36,5 @@ def initialize(backend)
     def xpath(*args)
       backend.xpath(*args).map { |node| backend.proxy_node.new(node) }
     end
-
-    # Searches elements by CSS selector.
-    #
-    # @return [Array<ProxyFetcher::Document::Node>]
-    #   collection of nodes
-    #
-    def css(*args)
-      backend.css(*args).map { |node| backend.proxy_node.new(node) }
-    end
   end
 end
diff --git a/lib/proxy_fetcher/document/adapters.rb b/lib/proxy_fetcher/document/adapters.rb
@@ -13,7 +13,7 @@ class Document
     # <code>ProxyFetcher::Document::AbstractAdapter</code>.
     class Adapters
       # Adapters class name suffix
-      ADAPTER = 'Adapter'.freeze
+      ADAPTER = "Adapter"
       private_constant :ADAPTER
 
       class << self

diff --git a/lib/proxy_fetcher/document/adapters/abstract_adapter.rb b/lib/proxy_fetcher/document/adapters/abstract_adapter.rb
@@ -26,23 +26,14 @@ def xpath(selector)
         document.xpath(selector)
       end
 
-      # You can override this method in your own adapter class
-      #
-      # @param selector [String]
-      #   CSS selector
-      #
-      def css(selector)
-        document.css(selector)
-      end
-
       # Returns <code>Node</code> class that will handle HTML
       # nodes for particular adapter.
       #
       # @return [ProxyFetcher::Document::Node]
       #   node
       #
       def proxy_node
-        self.class.const_get('Node')
+        self.class.const_get("Node")
       end
 
       # Installs adapter requirements.
@@ -53,8 +44,8 @@ def proxy_node
       def self.setup!(*args)
         install_requirements!(*args)
         self
-      rescue LoadError, StandardError => error
-        raise Exceptions::AdapterSetupError.new(name, error.message)
+      rescue LoadError, StandardError => e
+        raise Exceptions::AdapterSetupError.new(name, e.message)
       end
     end
   end

diff --git a/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb b/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb
@@ -6,7 +6,7 @@ class Document
     class NokogiriAdapter < AbstractAdapter
       # Requires Nokogiri gem to the application.
       def self.install_requirements!
-        require 'nokogiri'
+        require "nokogiri"
       end
 
       # Parses raw HTML content with specific gem.