[feature:lib] Add support for paging

README.md

@@ -108,6 +108,26 @@ Arx::Query.new(sort_by: :date_submitted, sort_order: :ascending)
 Arx::Query.new #=> sortBy=relevance&sortOrder=descending
 ```
 
+#### Paging
+
+The arXiv API offers a paging mechanism that allows you to get chucks of the result set at a time. It can be used through the `start` and `max_results` keyword arguments (in the `Arx::Query` initializer):
+
+- `start` is the index of the first returned result (using 0-based indexing)
+
+- `max_results` is the number of results returned by the query
+
+```ruby
+# Get results 10-29
+Arx::Query.new(start: 10, max_results: 20)
+#=> start=10&max_results=20
+```
+
+**Note**: The default values are those of the arXiv API: `start` defaults to `0` and `max_results` defaults to `10`:
+
+```ruby
+Arx::Query.new #=> start=0&max_results=10
+```
+
 #### Searching by ID
 
 The arXiv search API doesn't only support searching for papers by metadata fields, but also by ID. When searching by ID, a different URL query string parameter `id_list` is used (instead of `search_query` as seen before).

lib/arx.rb

@@ -47,19 +47,21 @@ class << self
 
     # Performs a search query for papers on the arXiv search API.
     #
-    # @note The +sort_by+ and +sort_order+ arguments are ignored if passing in your own +query+.
+    # @note The +sort_by+, +sort_order+, +start+ and +max_results+ arguments are ignored if passing in your own +query+.
     # @param ids [Array<String>] The IDs of the arXiv papers to restrict the query to.
     # @param query [Query, NilClass] Predefined search query object.
     # @param sort_by [Symbol] The sorting criteria for the returned results (see {Query::SORT_BY}).
     # @param sort_order [Symbol] The sorting order for the returned results (see {Query::SORT_ORDER}).
+    # @param start [Integer] The index of the first returned result.
+    # @param max_results [Integer] The number of results returned by the query
     # @return [Array<Paper>, Paper] The {Paper}(s) found by the search query.
-    def search(*ids, query: nil, sort_by: :relevance, sort_order: :descending)
-      query ||= Query.new(*ids, sort_by: sort_by, sort_order: sort_order)
+    def search(*ids, query: nil, sort_by: :relevance, sort_order: :descending, start: 0, max_results: 10)
+      query ||= Query.new(*ids, sort_by: sort_by, sort_order: sort_order, start: start, max_results: max_results)
       raise TypeError.new("Expected `query` to be an Arx::Query, got: #{query.class}") unless query.is_a? Query
 
       yield query if block_given?
 
-      document = Nokogiri::XML(URI.open ENDPOINT + query.to_s + '&max_results=10000').remove_namespaces!
+      document = Nokogiri::XML(URI.open ENDPOINT + query.to_s).remove_namespaces!
       results = Paper.parse(document, single: ids.size == 1)
 
       if results.is_a? Paper

lib/arx/query/query.rb

@@ -12,7 +12,9 @@ class Query
       search_query: 'search_query',
       id_list: 'id_list',
       sort_by: 'sortBy',
-      sort_order: 'sortOrder'
+      sort_order: 'sortOrder',
+      start: 'start',
+      max_results: 'max_results',
     }
 
     # Logical connectives supported by the arXiv search API.
@@ -55,8 +57,10 @@ class Query
     # @param ids [Array<String>] The IDs of the arXiv papers to restrict the query to.
     # @param sort_by [Symbol] The sorting criteria for the returned results (see {SORT_BY}).
     # @param sort_order [Symbol] The sorting order for the returned results (see {SORT_ORDER}).
+    # @param start [Integer] The index of the first returned result.
+    # @param max_results [Integer] The number of results returned by the query
     # @return [Query] The initialized query object.
-    def initialize(*ids, sort_by: :relevance, sort_order: :descending)
+    def initialize(*ids, sort_by: :relevance, sort_order: :descending, start: 0, max_results: 10)
       @query = String.new
 
       Validate.sort_by sort_by, permitted: SORT_BY.keys
@@ -65,6 +69,9 @@ def initialize(*ids, sort_by: :relevance, sort_order: :descending)
       Validate.sort_order sort_order, permitted: SORT_ORDER.keys
       @query << "&#{PARAMS[:sort_order]}=#{SORT_ORDER[sort_order]}"
 
+      Validate.paging start, max_results
+      @query << "&#{PARAMS[:start]}=#{start}&#{PARAMS[:max_results]}=#{max_results}"
+
       ids.flatten!
       unless ids.empty?
         ids.map! {|id| Cleaner.extract_id(id, version: true)}

lib/arx/query/validate.rb

@@ -29,6 +29,18 @@ def sort_order(value, permitted:)
         raise ArgumentError.new("Expected `sort_order` to be one of #{permitted}, got: #{value}") unless permitted.include? value
       end
 
+      # Validates the paging fields of the query string: +start+ and +max_results+.
+      #
+      # @param start [Integer] The start value to validate.
+      # @param max_results [Integer] The max_results value to validate.
+      # @raise
+      #   [TypeError] If the value of +start+ is not an +Integer+.
+      #   [TypeError] If the value of +max_results+ is not an +Integer+.
+      def paging(start, max_results)
+        raise TypeError.new("Expected `start` to be an Integer, got: #{start.class}") unless start.is_a? Integer
+        raise TypeError.new("Expected `max_results` to be an Integer, got: #{max_results.class}") unless max_results.is_a? Integer
+      end
+
       # Validates a list of arXiv paper identifiers.
       #
       # @param ids [Array<String>] The identifiers to validate.

spec/arx/query/query_spec.rb

@@ -1,24 +1,26 @@
 require 'spec_helper'
 
 describe Query do
+  let(:default_arguments) {'sortBy=relevance&sortOrder=descending&start=0&max_results=10'}
+
   context '.initialize' do
     subject { Query }
 
     it { is_expected.to respond_to(:new).with(0..1).arguments }
     it { is_expected.to respond_to(:new).with_unlimited_arguments }
 
     context 'with no arguments' do
-      it { expect(subject.new.to_s).to eq 'sortBy=relevance&sortOrder=descending' }
+      it { expect(subject.new.to_s).to eq default_arguments }
     end
     context 'with IDs' do
       context '1105.5379' do
-        it { expect(subject.new('1105.5379').to_s).to eq 'sortBy=relevance&sortOrder=descending&id_list=1105.5379' }
+        it { expect(subject.new('1105.5379').to_s).to eq "#{default_arguments}&id_list=1105.5379" }
       end
       context 'cond-mat/9609089' do
-        it { expect(subject.new('cond-mat/9609089').to_s).to eq 'sortBy=relevance&sortOrder=descending&id_list=cond-mat/9609089' }
+        it { expect(subject.new('cond-mat/9609089').to_s).to eq "#{default_arguments}&id_list=cond-mat/9609089" }
       end
       context '1105.5379, cond-mat/9609089 and cs/0003044' do
-        it { expect(subject.new(*%w[1105.5379 cond-mat/9609089 cs/0003044]).to_s).to eq 'sortBy=relevance&sortOrder=descending&id_list=1105.5379,cond-mat/9609089,cs/0003044' }
+        it { expect(subject.new(*%w[1105.5379 cond-mat/9609089 cs/0003044]).to_s).to eq "#{default_arguments}&id_list=1105.5379,cond-mat/9609089,cs/0003044" }
       end
     end
     context 'with key-word arguments' do
@@ -31,29 +33,55 @@
           it { expect { subject.new(sort_order: 'invalid') }.to raise_error TypeError }
           it { expect { subject.new(sort_order: :invalid) }.to raise_error ArgumentError }
         end
+        context :start do
+          it { expect { subject.new(start: 'zero') }.to raise_error TypeError }
+          it { expect { subject.new(start: :zero) }.to raise_error TypeError }
+          it { expect { subject.new(start: [3,78]) }.to raise_error TypeError }
+        end
+        context :max_results do
+          it { expect { subject.new(max_results: 'zero') }.to raise_error TypeError }
+          it { expect { subject.new(max_results: :zero) }.to raise_error TypeError }
+          it { expect { subject.new(max_results: [3,78]) }.to raise_error TypeError }
+        end
       end
       context '(valid)' do
         context :sort_by do
           Arx::Query::SORT_BY.each do |key, field|
-            it { expect(subject.new(sort_by: key).to_s).to eq "sortBy=#{field}&sortOrder=descending" }
+            it { expect(subject.new(sort_by: key).to_s).to eq "sortBy=#{field}&sortOrder=descending&start=0&max_results=10" }
           end
         end
         context :sort_order do
           Arx::Query::SORT_ORDER.each do |key, field|
-            it { expect(subject.new(sort_order: key).to_s).to eq "sortBy=relevance&sortOrder=#{field}" }
+            it { expect(subject.new(sort_order: key).to_s).to eq "sortBy=relevance&sortOrder=#{field}&start=0&max_results=10" }
           end
         end
+        context :start do
+          it { expect(subject.new(start: 23).to_s).to eq "sortBy=relevance&sortOrder=descending&start=23&max_results=10" }
+        end
+        context :max_results do
+          it { expect(subject.new(max_results: 10000).to_s).to eq "sortBy=relevance&sortOrder=descending&start=0&max_results=10000" }
+        end
+        context "start and max_results" do
+          it { expect(subject.new(start: 33, max_results: 500).to_s).to eq "sortBy=relevance&sortOrder=descending&start=33&max_results=500" }
+        end
         context "sort_by and sort_order" do
           Arx::Query::SORT_BY.each do |sort_by_key, sort_by_field|
             Arx::Query::SORT_ORDER.each do |sort_order_key, sort_order_field|
-              it { expect(subject.new(sort_by: sort_by_key, sort_order: sort_order_key).to_s).to eq "sortBy=#{sort_by_field}&sortOrder=#{sort_order_field}" }
+              it { expect(subject.new(sort_by: sort_by_key, sort_order: sort_order_key).to_s).to eq "sortBy=#{sort_by_field}&sortOrder=#{sort_order_field}&start=0&max_results=10" }
+            end
+          end
+        end
+        context "all" do
+          Arx::Query::SORT_BY.each do |sort_by_key, sort_by_field|
+            Arx::Query::SORT_ORDER.each do |sort_order_key, sort_order_field|
+              it { expect(subject.new(sort_by: sort_by_key, sort_order: sort_order_key, start: 13, max_results: 2).to_s).to eq "sortBy=#{sort_by_field}&sortOrder=#{sort_order_field}&start=13&max_results=2" }
             end
           end
         end
       end
     end
     context 'with IDs and key-word arguments' do
-      it { expect(subject.new('1105.5379', 'cond-mat/9609089', sort_by: :date_submitted, sort_order: :ascending).to_s).to eq 'sortBy=submittedDate&sortOrder=ascending&id_list=1105.5379,cond-mat/9609089' }
+      it { expect(subject.new('1105.5379', 'cond-mat/9609089', sort_by: :date_submitted, sort_order: :ascending).to_s).to eq 'sortBy=submittedDate&sortOrder=ascending&start=0&max_results=10&id_list=1105.5379,cond-mat/9609089' }
     end
   end
 
@@ -68,11 +96,11 @@
         end
       end
       context 'with a query string' do
-        it { expect(query.title('Test').send(connective).to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22Test%22+#{Query::CONNECTIVES[connective]}" }
+        it { expect(query.title('Test').send(connective).to_s).to eq "sortBy=relevance&sortOrder=descending&start=0&max_results=10&search_query=ti:%22Test%22+#{Query::CONNECTIVES[connective]}" }
       end
       context 'with connective already present' do
         Query::CONNECTIVES.keys.each do |existing|
-          it { expect(query.title('Test').send(existing).send(connective).to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22Test%22+#{Query::CONNECTIVES[existing]}" }
+          it { expect(query.title('Test').send(existing).send(connective).to_s).to eq "sortBy=relevance&sortOrder=descending&start=0&max_results=10&search_query=ti:%22Test%22+#{Query::CONNECTIVES[existing]}" }
         end
       end
     end
@@ -83,29 +111,29 @@
       let(:query) { Query.new }
 
       context 'without a query string' do
-        it { expect(query.send(field, 'cs.AI').to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=#{Query::FIELDS[field]}:%22cs.AI%22" }
+        it { expect(query.send(field, 'cs.AI').to_s).to eq "#{default_arguments}&search_query=#{Query::FIELDS[field]}:%22cs.AI%22" }
       end
       context 'without a prior connective' do
-        it { expect(query.title('test').send(field, 'cs.AI').to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22test%22+AND+#{Query::FIELDS[field]}:%22cs.AI%22" }
+        it { expect(query.title('test').send(field, 'cs.AI').to_s).to eq "#{default_arguments}&search_query=ti:%22test%22+AND+#{Query::FIELDS[field]}:%22cs.AI%22" }
       end
       context 'with a prior connective' do
         Query::CONNECTIVES.keys.each do |connective|
-          it { expect(query.title('test').send(connective).send(field, 'cs.AI').to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22test%22+#{Query::CONNECTIVES[connective]}+#{Query::FIELDS[field]}:%22cs.AI%22" }
+          it { expect(query.title('test').send(connective).send(field, 'cs.AI').to_s).to eq "#{default_arguments}&search_query=ti:%22test%22+#{Query::CONNECTIVES[connective]}+#{Query::FIELDS[field]}:%22cs.AI%22" }
         end
       end
       context 'exact: false' do
-        it { expect(query.send(field, 'cs.AI', exact: false).to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=#{Query::FIELDS[field]}:cs.AI" }
+        it { expect(query.send(field, 'cs.AI', exact: false).to_s).to eq "#{default_arguments}&search_query=#{Query::FIELDS[field]}:cs.AI" }
       end
       context 'with multiple values' do
-        it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG').to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:%22cs.AI%22+AND+#{Query::FIELDS[field]}:%22cs.LG%22%29" }
+        it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG').to_s).to eq "#{default_arguments}&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:%22cs.AI%22+AND+#{Query::FIELDS[field]}:%22cs.LG%22%29" }
 
         context 'exact: false' do
-          it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG', exact: false).to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:cs.AI+AND+#{Query::FIELDS[field]}:cs.LG%29" }
+          it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG', exact: false).to_s).to eq "#{default_arguments}&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:cs.AI+AND+#{Query::FIELDS[field]}:cs.LG%29" }
         end
 
         Query::CONNECTIVES.keys.each do |connective|
           context "connective: #{connective}" do
-            it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG', connective: connective).to_s).to eq "sortBy=relevance&sortOrder=descending&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:%22cs.AI%22+#{Query::CONNECTIVES[connective]}+#{Query::FIELDS[field]}:%22cs.LG%22%29" }
+            it { expect(query.title('test').send(field, 'cs.AI', 'cs.LG', connective: connective).to_s).to eq "#{default_arguments}&search_query=ti:%22test%22+AND+%28#{Query::FIELDS[field]}:%22cs.AI%22+#{Query::CONNECTIVES[connective]}+#{Query::FIELDS[field]}:%22cs.LG%22%29" }
           end
         end
       end
@@ -116,18 +144,18 @@
     subject { Query }
 
     context 'with no search query' do
-      it { expect(subject.new.group {}.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=%28%29' }
+      it { expect(subject.new.group {}.to_s).to eq "#{default_arguments}&search_query=%28%29" }
       it do
         query = subject.new.tap do |q|
           q.group { q.title 'Buchi automata' }
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=%28ti:%22Buchi+automata%22%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=%28ti:%22Buchi+automata%22%29"
       end
       it do
         query = subject.new.tap do |q|
           q.group { q.group { q.group {} } }
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=%28+%28+%28%29%29%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=%28+%28+%28%29%29%29"
       end
     end
     context 'with no block' do
@@ -146,7 +174,7 @@
             q.author 'Tomáš Babiak'
           end
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=ti:%22Buchi+automata%22+AND+%28cat:%22cs.FL%22+ANDNOT+au:%22Tom%C3%A1%C5%A1+Babiak%22%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=ti:%22Buchi+automata%22+AND+%28cat:%22cs.FL%22+ANDNOT+au:%22Tom%C3%A1%C5%A1+Babiak%22%29"
       end
       it do
         query = subject.new.tap do |q|
@@ -157,7 +185,7 @@
             q.author 'Tomáš Babiak'
           end
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=ti:%22Buchi+automata%22+AND+%28%28cat:%22cs.FL%22+OR+cat:%22cs.CC%22%29+ANDNOT+au:%22Tom%C3%A1%C5%A1+Babiak%22%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=ti:%22Buchi+automata%22+AND+%28%28cat:%22cs.FL%22+OR+cat:%22cs.CC%22%29+ANDNOT+au:%22Tom%C3%A1%C5%A1+Babiak%22%29"
       end
       it do
         query = subject.new.tap do |q|
@@ -168,7 +196,7 @@
             q.category 'cs.FL', 'cs.CC', connective: :or
           end
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=ti:%22Buchi+automata%22+AND+%28au:%22Tom%C3%A1%C5%A1+Babiak%22+OR+%28cat:%22cs.FL%22+OR+cat:%22cs.CC%22%29%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=ti:%22Buchi+automata%22+AND+%28au:%22Tom%C3%A1%C5%A1+Babiak%22+OR+%28cat:%22cs.FL%22+OR+cat:%22cs.CC%22%29%29"
       end
       it do
         query = subject.new.tap do |q|
@@ -177,7 +205,7 @@
             q.category 'cs.FL', 'cs.CC', connective: :and_not
           end
         end
-        expect(query.to_s).to eq 'sortBy=relevance&sortOrder=descending&search_query=ti:%22Buchi+automata%22+AND+%28%28cat:%22cs.FL%22+ANDNOT+cat:%22cs.CC%22%29%29'
+        expect(query.to_s).to eq "#{default_arguments}&search_query=ti:%22Buchi+automata%22+AND+%28%28cat:%22cs.FL%22+ANDNOT+cat:%22cs.CC%22%29%29"
       end
     end
   end