-
Notifications
You must be signed in to change notification settings - Fork 78
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improved version of UniqueList: OrderedSet (#76)
* Back UniqueList with a Redis Sorted Set A sorted set is optimal for this type of functionality because we can use fewer, more-performant Redis calls. * Wrap unique list insert and trim in a transaction * Fallback / migration mechanism for existing users of UniqueLists In order to ease the transition from UniqueList being backed by a Redis list, we can fallback to the legacy implementation for read operations. For write operations we first migrate to a sorted set, then retry. * Update the Readme with sorted set backed unique list commands * Test remaining methods with migration fallbacks * Be more specific about which errors to raise * Improve the fallback / migration system * Improve unique list scoring to avoid collisions Use the Redis server time as a basis. Then add the current process time. Then add to that the index of the inserted elements as a microsecond component. * Update Readme with realistic Unique List scores * Implement OrderedSet Like a UniqueList, but backed by a Redis sorted set rather than a list * Resolve pipeline deprecation warnings * Document OrderedSet * Remove UniqueListLegacy * Revert fallback options * Revert change to type_from * Enforce limit is not negative * Revert config proxying * Fix bug in score ge neration
- Loading branch information
Showing
4 changed files
with
183 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
class Kredis::Types::OrderedSet < Kredis::Types::Proxying | ||
proxying :multi, :zrange, :zrem, :zadd, :zremrangebyrank, :zcard, :exists?, :del | ||
|
||
attr_accessor :typed | ||
attr_reader :limit | ||
|
||
def elements | ||
strings_to_types(zrange(0, -1) || [], typed) | ||
end | ||
alias to_a elements | ||
|
||
def remove(*elements) | ||
zrem(types_to_strings(elements, typed)) | ||
end | ||
|
||
def prepend(elements) | ||
insert(elements, prepending: true) | ||
end | ||
|
||
def append(elements) | ||
insert(elements) | ||
end | ||
alias << append | ||
|
||
def limit=(limit) | ||
raise "Limit must be greater than 0" if limit && limit <= 0 | ||
|
||
@limit = limit | ||
end | ||
|
||
private | ||
def insert(elements, prepending: false) | ||
elements = Array(elements) | ||
return if elements.empty? | ||
|
||
elements_with_scores = types_to_strings(elements, typed).map.with_index do |element, index| | ||
score = generate_base_score(negative: prepending) + (index / 100000) | ||
|
||
[ score , element ] | ||
end | ||
|
||
multi do |pipeline| | ||
pipeline.zadd(elements_with_scores) | ||
trim(from_beginning: prepending, pipeline: pipeline) | ||
end | ||
end | ||
|
||
def generate_base_score(negative:) | ||
current_time = process_start_time + process_uptime | ||
|
||
negative ? -current_time : current_time | ||
end | ||
|
||
def process_start_time | ||
@process_start_time ||= redis.time.join(".").to_f - process_uptime | ||
end | ||
|
||
def process_uptime | ||
Process.clock_gettime(Process::CLOCK_MONOTONIC) | ||
end | ||
|
||
def trim(from_beginning:, pipeline:) | ||
return unless limit | ||
|
||
if from_beginning | ||
pipeline.zremrangebyrank(limit, -1) | ||
else | ||
pipeline.zremrangebyrank(0, -(limit + 1)) | ||
end | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
require "test_helper" | ||
|
||
class OrderedSetTest < ActiveSupport::TestCase | ||
setup { @set = Kredis.ordered_set "ordered-set", limit: 5 } | ||
|
||
test "append" do | ||
@set.append(%w[ 1 2 3 ]) | ||
@set.append(%w[ 1 2 3 4 ]) | ||
assert_equal %w[ 1 2 3 4 ], @set.elements | ||
|
||
@set << "5" | ||
assert_equal %w[ 1 2 3 4 5 ], @set.elements | ||
end | ||
|
||
test "appending the same element re-appends it" do | ||
@set.append(%w[ 1 2 3 ]) | ||
@set.append(%w[ 2 ]) | ||
assert_equal %w[ 1 3 2 ], @set.elements | ||
end | ||
|
||
test "mass append maintains ordering" do | ||
@set = Kredis.ordered_set "ordered-set" # no limit | ||
|
||
thousand_elements = 1000.times.map { [*"A".."Z"].sample(10).join } | ||
@set.append(thousand_elements) | ||
assert_equal thousand_elements, @set.elements | ||
|
||
thousand_elements.each { |element| @set.append(element) } | ||
assert_equal thousand_elements, @set.elements | ||
end | ||
|
||
test "prepend" do | ||
@set.prepend(%w[ 1 2 3 ]) | ||
@set.prepend(%w[ 1 2 3 4 ]) | ||
assert_equal %w[ 4 3 2 1 ], @set.elements | ||
end | ||
|
||
test "append nothing" do | ||
@set.append(%w[ 1 2 3 ]) | ||
@set.append([]) | ||
assert_equal %w[ 1 2 3 ], @set.elements | ||
end | ||
|
||
test "prepend nothing" do | ||
@set.prepend(%w[ 1 2 3 ]) | ||
@set.prepend([]) | ||
assert_equal %w[ 3 2 1 ], @set.elements | ||
end | ||
|
||
test "typed as integers" do | ||
@set = Kredis.ordered_set "mylist", typed: :integer | ||
|
||
@set.append [ 1, 2 ] | ||
@set << 2 | ||
assert_equal [ 1, 2 ], @set.elements | ||
|
||
@set.remove(2) | ||
assert_equal [ 1 ], @set.elements | ||
|
||
@set.append [ "1-a", 2 ] | ||
|
||
assert_equal [ 1, 2 ], @set.elements | ||
end | ||
|
||
test "exists?" do | ||
assert_not @set.exists? | ||
|
||
@set.append [ 1, 2 ] | ||
assert @set.exists? | ||
end | ||
|
||
test "appending over limit" do | ||
@set.append(%w[ 1 2 3 4 5 ]) | ||
@set.append(%w[ 6 7 8 ]) | ||
assert_equal %w[ 4 5 6 7 8 ], @set.elements | ||
end | ||
|
||
test "prepending over limit" do | ||
@set.prepend(%w[ 1 2 3 4 5 ]) | ||
@set.prepend(%w[ 6 7 8 ]) | ||
assert_equal %w[ 8 7 6 5 4 ], @set.elements | ||
end | ||
|
||
test "appending array with duplicates" do | ||
@set.append(%w[ 1 1 1 ]) | ||
assert_equal %w[ 1 ], @set.elements | ||
end | ||
|
||
test "prepending array with duplicates" do | ||
@set.prepend(%w[ 1 1 1 ]) | ||
assert_equal %w[ 1 ], @set.elements | ||
end | ||
|
||
test "limit can't be 0 or less" do | ||
assert_raises do | ||
Kredis.ordered_set "ordered-set", limit: -1 | ||
end | ||
end | ||
end |