-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
17 changed files
with
335 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package key | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/binary" | ||
| "errors" | ||
|
|
||
| "github.com/rotationalio/honu/pkg/store/lamport" | ||
| "go.rtnl.ai/ulid" | ||
| ) | ||
|
|
||
| const ( | ||
| // The default size of a v1 object storage key. | ||
| keySize int = 45 | ||
|
|
||
| // The version of the key for compatibility indication; increment this number any time | ||
| // the underlying key data is no longer compatible with the previous version. | ||
| keyVersion byte = 0x1 | ||
| ) | ||
|
|
||
| var ( | ||
| ErrBadVersion = errors.New("key is malformed: cannot decode specified version") | ||
| ErrBadSize = errors.New("key is malformed: incorrect size") | ||
| ErrMalformed = errors.New("key is malformed: cannot parse version components") | ||
| ) | ||
|
|
||
| // Keys are used to store objects in the underlying key/value store. It is a 45 byte key | ||
| // that is composed of 16 byte object and collection IDs and a 4 byte uint32 and 8 byte | ||
| // uint64 representing the lamport scalar version number. The last byte indicates the | ||
| // key version and marshaling compatibility. There are no separator characters | ||
| // between the components of the key since all components are a fixed length. | ||
| // | ||
| // A key is structured as: collection::oid::vid::pid::keyVersion | ||
| // | ||
| // Note that the version is serialized differently than the lamport scalar in order to | ||
| // maintain lexicographic sorting of the the data. | ||
| type Key []byte | ||
|
|
||
| func New(oid, cid ulid.ULID, vers lamport.Scalar) Key { | ||
| key := make([]byte, keySize) | ||
| copy(key[0:16], oid[:]) | ||
| copy(key[16:32], cid[:]) | ||
| binary.BigEndian.PutUint64(key[32:40], vers.VID) | ||
| binary.BigEndian.PutUint32(key[40:44], vers.PID) | ||
| key[44] = keyVersion | ||
| return Key(key) | ||
| } | ||
|
|
||
| func (k Key) ObjectID() ulid.ULID { | ||
| if err := k.Check(); err != nil { | ||
| panic(err) | ||
| } | ||
| return ulid.ULID(k[0:16]) | ||
| } | ||
|
|
||
| func (k Key) CollectionID() ulid.ULID { | ||
| if err := k.Check(); err != nil { | ||
| panic(err) | ||
| } | ||
| return ulid.ULID(k[16:32]) | ||
| } | ||
|
|
||
| func (k Key) Version() lamport.Scalar { | ||
| if err := k.Check(); err != nil { | ||
| panic(err) | ||
| } | ||
| return lamport.Scalar{ | ||
| VID: binary.BigEndian.Uint64(k[32:40]), | ||
| PID: binary.BigEndian.Uint32(k[40:44]), | ||
| } | ||
| } | ||
|
|
||
| func (k Key) Check() error { | ||
| if len(k) != keySize { | ||
| return ErrBadSize | ||
| } | ||
|
|
||
| if k[44] != keyVersion { | ||
| return ErrBadVersion | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| //=========================================================================== | ||
| // Sort Interface | ||
| //=========================================================================== | ||
|
|
||
| type Keys []Key | ||
|
|
||
| func (k Keys) Len() int { return len(k) } | ||
| func (k Keys) Less(i, j int) bool { return bytes.Compare(k[i][:], k[j][:]) < 0 } | ||
| func (k Keys) Swap(i, j int) { k[i], k[j] = k[j], k[i] } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| package key_test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| crand "crypto/rand" | ||
| "math/rand/v2" | ||
| "sort" | ||
| "testing" | ||
|
|
||
| "github.com/rotationalio/honu/pkg/store/key" | ||
| "github.com/rotationalio/honu/pkg/store/lamport" | ||
| "github.com/stretchr/testify/require" | ||
| "go.rtnl.ai/ulid" | ||
| ) | ||
|
|
||
| func TestNewKey(t *testing.T) { | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := lamport.Scalar{VID: 1, PID: 2} | ||
|
|
||
| k := key.New(oid, cid, vers) | ||
| require.NotNil(t, k) | ||
| require.Equal(t, 45, len(k)) | ||
| require.Equal(t, oid, k.ObjectID()) | ||
| require.Equal(t, cid, k.CollectionID()) | ||
| require.Equal(t, vers, k.Version()) | ||
| } | ||
|
|
||
| func TestKeyLexicographic(t *testing.T) { | ||
| // For the same collection and object ID the keys should be lexicographically sorted | ||
| // by the version to ensure that we can read the latest version either by choosing | ||
| // the first item in a list of sorted keys or the last item. | ||
| keys := make(key.Keys, 512) | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := &lamport.Scalar{VID: 1, PID: 1} | ||
|
|
||
| // Create a list of keys with monotonically increasing versions. | ||
| for i := 0; i < len(keys); i++ { | ||
| keys[i] = key.New(oid, cid, *vers) | ||
| vers = randNextScalar(vers) | ||
| } | ||
|
|
||
| // Ensure the keys are sorted both by monotonically increasing version and by | ||
| // lexicographic byte order. | ||
| for i := 1; i < len(keys); i++ { | ||
| versa, versb := keys[i-1].Version(), keys[i].Version() | ||
| require.True(t, versa.Before(&versb) || versa.Equals(&versb), "keys[%d] version is not before keys[%d] version", i-1, i) | ||
| require.True(t, bytes.Compare(keys[i-1][:], keys[i][:]) <= 0, "keys[%d] is not less than or equal to keys[%d]", i-1, i) | ||
| } | ||
| } | ||
|
|
||
| func TestKeyCheck(t *testing.T) { | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := lamport.Scalar{VID: 1, PID: 2} | ||
|
|
||
| t.Run("Valid", func(t *testing.T) { | ||
| k := key.New(oid, cid, vers) | ||
| require.NoError(t, k.Check()) | ||
| }) | ||
|
|
||
| t.Run("BadSize", func(t *testing.T) { | ||
| badKey := key.Key(make([]byte, 42)) | ||
| require.ErrorIs(t, badKey.Check(), key.ErrBadSize) | ||
| }) | ||
|
|
||
| t.Run("BadVersion", func(t *testing.T) { | ||
| badKey := key.New(oid, cid, vers) | ||
| badKey[44] = 0x2 | ||
| require.ErrorIs(t, badKey.Check(), key.ErrBadVersion) | ||
| }) | ||
| } | ||
|
|
||
| func TestObjectID(t *testing.T) { | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := lamport.Scalar{VID: 80, PID: 122} | ||
|
|
||
| t.Run("Ok", func(t *testing.T) { | ||
| k := key.New(oid, cid, vers) | ||
| require.Equal(t, oid, k.ObjectID()) | ||
| }) | ||
|
|
||
| t.Run("Panics", func(t *testing.T) { | ||
| badKey := key.Key(make([]byte, 42)) | ||
| require.Panics(t, func() { | ||
| badKey.ObjectID() | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func TestCollectionID(t *testing.T) { | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := lamport.Scalar{VID: 391, PID: 8} | ||
|
|
||
| t.Run("Ok", func(t *testing.T) { | ||
| k := key.New(oid, cid, vers) | ||
| require.Equal(t, cid, k.CollectionID()) | ||
| }) | ||
|
|
||
| t.Run("Panics", func(t *testing.T) { | ||
| badKey := key.Key(make([]byte, 42)) | ||
| require.Panics(t, func() { | ||
| badKey.CollectionID() | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func TestVersion(t *testing.T) { | ||
| oid := ulid.Make() | ||
| cid := ulid.Make() | ||
| vers := lamport.Scalar{VID: 5, PID: 1} | ||
|
|
||
| t.Run("Ok", func(t *testing.T) { | ||
| k := key.New(oid, cid, vers) | ||
| require.Equal(t, vers, k.Version()) | ||
| }) | ||
|
|
||
| t.Run("Panics", func(t *testing.T) { | ||
| badKey := key.Key(make([]byte, 42)) | ||
| require.Panics(t, func() { | ||
| badKey.Version() | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func TestSort(t *testing.T) { | ||
| keys := make(key.Keys, 128) | ||
| for i := 0; i < 128; i++ { | ||
| keys[i] = randKey() | ||
| } | ||
|
|
||
| sort.Sort(keys) | ||
|
|
||
| for i := 1; i < len(keys); i++ { | ||
| require.True(t, bytes.Compare(keys[i-1][:], keys[i][:]) <= 0, "keys[%d] is not less than or equal to keys[%d]", i-1, i) | ||
| } | ||
| } | ||
|
|
||
| func randKey() key.Key { | ||
| b := make([]byte, 45) | ||
| crand.Read(b) | ||
| return key.Key(b) | ||
| } | ||
|
|
||
| func randNextScalar(prev *lamport.Scalar) *lamport.Scalar { | ||
| s := &lamport.Scalar{} | ||
| s.PID = uint32(rand.Int32N(24)) | ||
| s.VID = uint64(rand.Int64N(32)) + prev.VID | ||
| return s | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,7 +5,7 @@ import ( | |
| "reflect" | ||
| "time" | ||
|
|
||
| "github.com/oklog/ulid/v2" | ||
| "go.rtnl.ai/ulid" | ||
| ) | ||
|
|
||
| const ( | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.