Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved version of UniqueList: OrderedSet #76

Merged
merged 19 commits into from
Jun 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ unique_list << "5" # => LREM myuniquelist 0, "5" + R
unique_list.remove(3) # => LREM myuniquelist 0, "3"
[ "4", "2", "1", "5" ] == unique_list.elements # => LRANGE myuniquelist 0, -1

ordered_set = Kredis.ordered_set "myorderedset"
ordered_set.append(%w[ 2 3 4 ]) # => ZADD myorderedset 1646131025.4953232 2 1646131025.495326 3 1646131025.4953272 4
ordered_set.prepend(%w[ 1 2 3 4 ]) # => ZADD myorderedset -1646131025.4957051 1 -1646131025.495707 2 -1646131025.4957082 3 -1646131025.4957092 4
ordered_set.append([])
ordered_set << "5" # => ZADD myorderedset 1646131025.4960442 5
ordered_set.remove(3) # => ZREM myorderedset 3
[ "4", "2", "1", "5" ] == ordered_set.elements # => ZRANGE myorderedset 0 -1

set = Kredis.set "myset", typed: :datetime
set.add(DateTime.tomorrow, DateTime.yesterday) # => SADD myset "2021-02-03 00:00:00 +0100" "2021-02-01 00:00:00 +0100"
set << DateTime.tomorrow # => SADD myset "2021-02-03 00:00:00 +0100"
Expand Down
5 changes: 5 additions & 0 deletions lib/kredis/types.rb
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ def set(key, typed: :string, config: :shared, after_change: nil)
type_from(Set, config, key, after_change: after_change, typed: typed)
end

def ordered_set(key, typed: :string, limit: nil, config: :shared, after_change: nil)
type_from(OrderedSet, config, key, after_change: after_change, typed: typed, limit: limit)
end

def slot(key, config: :shared, after_change: nil)
type_from(Slots, config, key, after_change: after_change, available: 1)
end
Expand Down Expand Up @@ -99,4 +103,5 @@ def type_from(type_klass, config, key, after_change: nil, **options)
require "kredis/types/list"
require "kredis/types/unique_list"
require "kredis/types/set"
require "kredis/types/ordered_set"
require "kredis/types/slots"
71 changes: 71 additions & 0 deletions lib/kredis/types/ordered_set.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
class Kredis::Types::OrderedSet < Kredis::Types::Proxying
proxying :multi, :zrange, :zrem, :zadd, :zremrangebyrank, :zcard, :exists?, :del

attr_accessor :typed
attr_reader :limit

def elements
strings_to_types(zrange(0, -1) || [], typed)
end
alias to_a elements

def remove(*elements)
zrem(types_to_strings(elements, typed))
end

def prepend(elements)
insert(elements, prepending: true)
end

def append(elements)
insert(elements)
end
alias << append

def limit=(limit)
raise "Limit must be greater than 0" if limit && limit <= 0

@limit = limit
end

private
def insert(elements, prepending: false)
elements = Array(elements)
return if elements.empty?

elements_with_scores = types_to_strings(elements, typed).map.with_index do |element, index|
score = generate_base_score(negative: prepending) + (index / 100000)

[ score , element ]
end

multi do |pipeline|
pipeline.zadd(elements_with_scores)
trim(from_beginning: prepending, pipeline: pipeline)
end
end

def generate_base_score(negative:)
current_time = process_start_time + process_uptime

negative ? -current_time : current_time
end
Comment on lines +48 to +52
Copy link
Contributor

@intrip intrip May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Clever approach 👍!

Just a note for your consideration:

What would happen in case of skewed Redis clocks / concurrent writes? Is possible to face the scenario where appending 2 arrays would lead to mixing their elements?

# In process A
ordered_set.append([1, 2, 3])
# In process B
ordered_set.append([4, 5, 6])
# Is this possible?
ordered_set.elements
# => [1, 2, 4, 3, 5, 6]

Maybe a less clever approach that relies getting the max/min (with an optimistic lock) would be more resilient?

WATCH zset
# get max
element = ZRANGE zset -1 -1
# compute scores max +1 for each element
MULTI
# add elements
ZADD zset elements
# trim
EXEC


def process_start_time
@process_start_time ||= redis.time.join(".").to_f - process_uptime
end

def process_uptime
Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

def trim(from_beginning:, pipeline:)
return unless limit

if from_beginning
pipeline.zremrangebyrank(limit, -1)
else
pipeline.zremrangebyrank(0, -(limit + 1))
end
end
end
99 changes: 99 additions & 0 deletions test/types/ordered_set_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
require "test_helper"

class OrderedSetTest < ActiveSupport::TestCase
setup { @set = Kredis.ordered_set "ordered-set", limit: 5 }

test "append" do
@set.append(%w[ 1 2 3 ])
@set.append(%w[ 1 2 3 4 ])
assert_equal %w[ 1 2 3 4 ], @set.elements

@set << "5"
assert_equal %w[ 1 2 3 4 5 ], @set.elements
end

test "appending the same element re-appends it" do
@set.append(%w[ 1 2 3 ])
@set.append(%w[ 2 ])
assert_equal %w[ 1 3 2 ], @set.elements
end

test "mass append maintains ordering" do
@set = Kredis.ordered_set "ordered-set" # no limit

thousand_elements = 1000.times.map { [*"A".."Z"].sample(10).join }
@set.append(thousand_elements)
assert_equal thousand_elements, @set.elements

thousand_elements.each { |element| @set.append(element) }
assert_equal thousand_elements, @set.elements
end

test "prepend" do
@set.prepend(%w[ 1 2 3 ])
@set.prepend(%w[ 1 2 3 4 ])
assert_equal %w[ 4 3 2 1 ], @set.elements
end

test "append nothing" do
@set.append(%w[ 1 2 3 ])
@set.append([])
assert_equal %w[ 1 2 3 ], @set.elements
end

test "prepend nothing" do
@set.prepend(%w[ 1 2 3 ])
@set.prepend([])
assert_equal %w[ 3 2 1 ], @set.elements
end

test "typed as integers" do
@set = Kredis.ordered_set "mylist", typed: :integer

@set.append [ 1, 2 ]
@set << 2
assert_equal [ 1, 2 ], @set.elements

@set.remove(2)
assert_equal [ 1 ], @set.elements

@set.append [ "1-a", 2 ]

assert_equal [ 1, 2 ], @set.elements
end

test "exists?" do
assert_not @set.exists?

@set.append [ 1, 2 ]
assert @set.exists?
end

test "appending over limit" do
@set.append(%w[ 1 2 3 4 5 ])
@set.append(%w[ 6 7 8 ])
assert_equal %w[ 4 5 6 7 8 ], @set.elements
end

test "prepending over limit" do
@set.prepend(%w[ 1 2 3 4 5 ])
@set.prepend(%w[ 6 7 8 ])
assert_equal %w[ 8 7 6 5 4 ], @set.elements
end

test "appending array with duplicates" do
@set.append(%w[ 1 1 1 ])
assert_equal %w[ 1 ], @set.elements
end

test "prepending array with duplicates" do
@set.prepend(%w[ 1 1 1 ])
assert_equal %w[ 1 ], @set.elements
end

test "limit can't be 0 or less" do
assert_raises do
Kredis.ordered_set "ordered-set", limit: -1
end
end
end