diff --git a/Cargo.toml b/Cargo.toml index 89e7d1ca..223cd26c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,42 +2,38 @@ name = "agatedb" version = "0.1.0" authors = ["Jay Lee "] -edition = "2018" +edition = "2021" [dependencies] -bytes = "1.0" +bytes = "1.1.0" coarsetime = "0.1.22" -crc = "1.8" -crc32fast = "1.2" -crossbeam-channel = "0.5" -enum_dispatch = "0.3" -farmhash = "1.1" +crc = "3.0.0" +crc32fast = "1.3.2" +crossbeam-channel = "0.5.5" +enum_dispatch = "0.3.8" +farmhash = "1.1.5" getset = "0.1.2" lazy_static = "1.4.0" log = "0.4.17" -memmap2 = "0.3" -parking_lot = "0.11" -prost = "0.8" +memmap2 = "0.5.4" +parking_lot = "0.12.1" +prost = "0.10.4" proto = { path = "proto" } -rand = "0.7" +rand = "0.8.5" skiplist = { path = "skiplist" } -tempdir = "0.3" -thiserror = "1.0" +tempdir = "0.3.7" +thiserror = "1.0.31" yatp = { git = "https://github.com/tikv/yatp.git" } [dev-dependencies] -criterion = "0.3" -tempfile = "3" +criterion = "0.3.5" +tempfile = "3.3.0" [target.'cfg(not(target_env = "msvc"))'.dev-dependencies] -tikv-jemallocator = "0.4.0" +tikv-jemallocator = "0.5.0" [workspace] -members = [ - "agate_bench", - "proto", - "skiplist" -] +members = ["agate_bench", "proto", "skiplist"] [[bench]] name = "bench_common" diff --git a/agate_bench/Cargo.toml b/agate_bench/Cargo.toml index 4b3faea8..0dd0da42 100644 --- a/agate_bench/Cargo.toml +++ b/agate_bench/Cargo.toml @@ -19,4 +19,4 @@ threadpool = "1.8" yatp = { git = "https://github.com/tikv/yatp.git" } [target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.4.0" +tikv-jemallocator = "0.5.0" diff --git a/agate_bench/src/main.rs b/agate_bench/src/main.rs index 7853c576..d4ab9a85 100644 --- a/agate_bench/src/main.rs +++ b/agate_bench/src/main.rs @@ -1,13 +1,14 @@ +use std::{ + path::PathBuf, + sync::{mpsc::channel, Arc}, + time::{Duration, UNIX_EPOCH}, +}; + use agatedb::{AgateOptions, IteratorOptions}; use bytes::{Bytes, BytesMut}; use clap::clap_app; use indicatif::{ProgressBar, ProgressStyle}; use rand::Rng; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::UNIX_EPOCH; -use std::{sync::mpsc::channel, time::Duration}; - #[cfg(not(target_env = "msvc"))] use tikv_jemallocator::Jemalloc; @@ -169,7 +170,7 @@ fn main() { let (key, value) = if seq { gen_kv_pair(j, value_size) } else { - gen_kv_pair(rng.gen_range(0, key_nums), value_size) + gen_kv_pair(rng.gen_range(0..key_nums), value_size) }; txn.set(key, value).unwrap(); write.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -236,7 +237,7 @@ fn main() { let txn = agate.new_transaction_at(unix_time(), false); let mut rng = rand::thread_rng(); for _ in range { - let (key, _) = gen_kv_pair(rng.gen_range(0, key_nums), value_size); + let (key, _) = gen_kv_pair(rng.gen_range(0..key_nums), value_size); match txn.get(&key) { Ok(item) => { assert_eq!(item.value().len(), value_size); @@ -369,7 +370,7 @@ fn main() { let (key, value) = if seq { gen_kv_pair(j, value_size) } else { - gen_kv_pair(rng.gen_range(0, key_nums), value_size) + gen_kv_pair(rng.gen_range(0..key_nums), value_size) }; batch.put(key, value); write.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -431,7 +432,7 @@ fn main() { let range = (i * chunk_size)..((i + 1) * chunk_size); let mut rng = rand::thread_rng(); for _ in range { - let (key, _) = gen_kv_pair(rng.gen_range(0, key_nums), value_size); + let (key, _) = gen_kv_pair(rng.gen_range(0..key_nums), value_size); match db.get(&key) { Ok(Some(value)) => { assert_eq!(value.len(), value_size); diff --git a/benches/bench_iterator.rs b/benches/bench_iterator.rs index 692c5483..44f07c08 100644 --- a/benches/bench_iterator.rs +++ b/benches/bench_iterator.rs @@ -1,12 +1,8 @@ mod common; -use agatedb::util::unix_time; -use agatedb::AgateIterator; -use agatedb::AgateOptions; -use agatedb::ConcatIterator; -use agatedb::Iterators; -use agatedb::MergeIterator; - +use agatedb::{ + util::unix_time, AgateIterator, AgateOptions, ConcatIterator, Iterators, MergeIterator, +}; use bytes::Bytes; use common::get_table_for_benchmark; use criterion::{criterion_group, criterion_main, Criterion}; @@ -58,7 +54,7 @@ fn bench_iterator(c: &mut Criterion) { c.bench_function("iterate noprefix single key", |b| { let txn = db.new_transaction_at(unix_time(), false); - let key_id = thread_rng().gen_range(0, N); + let key_id = thread_rng().gen_range(0..N); let seek_key = key(key_id); let it_opts = agatedb::IteratorOptions { all_versions: true, @@ -110,7 +106,7 @@ fn bench_merge_iterator(c: &mut Criterion) { c.bench_function("merge iterator random read", |b| { b.iter_batched( || { - let i = rng.gen_range(0, n); + let i = rng.gen_range(0..n); ( Bytes::from(format!("{:016x}", i)), Bytes::from(i.to_string()), @@ -149,7 +145,7 @@ fn bench_concat_iterator(c: &mut Criterion) { c.bench_function("concat iterator random read", |b| { b.iter_batched( || { - let i = rng.gen_range(0, n); + let i = rng.gen_range(0..n); ( Bytes::from(format!("{:016x}", i)), Bytes::from(i.to_string()), diff --git a/benches/bench_table.rs b/benches/bench_table.rs index 69e66be6..62f09f5e 100644 --- a/benches/bench_table.rs +++ b/benches/bench_table.rs @@ -89,7 +89,7 @@ fn bench_table(c: &mut Criterion) { b.iter_batched( || { - let i = rng.gen_range(0, n); + let i = rng.gen_range(0..n); ( Bytes::from(format!("{:016x}", i)), Bytes::from(i.to_string()), diff --git a/benches/common.rs b/benches/common.rs index 6742ad2f..af350b82 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -1,17 +1,19 @@ #![allow(dead_code)] +use std::ops::{Deref, DerefMut}; + use agatedb::{ opt::build_table_options, AgateOptions, ChecksumVerificationMode::NoVerification, Table, TableBuilder, Value, }; use bytes::Bytes; use rand::{distributions::Alphanumeric, Rng}; -use std::ops::{Deref, DerefMut}; use tempdir::TempDir; pub fn rand_value() -> String { rand::thread_rng() .sample_iter(&Alphanumeric) .take(32) + .map(char::from) .collect::() } diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 1f55de7e..f0ecd460 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -2,11 +2,11 @@ name = "proto" version = "0.1.0" authors = ["Jay Lee "] -edition = "2018" +edition = "2021" [dependencies] -bytes = "1.0" -prost = "0.8" +bytes = "1.1.0" +prost = "0.10.4" [build-dependencies] -prost-build = { version = "0.8" } +prost-build = "0.10.4" diff --git a/skiplist/Cargo.toml b/skiplist/Cargo.toml index 3db51e64..4beb97fc 100644 --- a/skiplist/Cargo.toml +++ b/skiplist/Cargo.toml @@ -2,18 +2,18 @@ name = "skiplist" version = "0.1.0" authors = ["Jay Lee "] -edition = "2018" +edition = "2021" [dependencies] -bytes = "1.0" -rand = "0.7" +bytes = "1.1.0" +rand = "0.8.5" [dev-dependencies] -criterion = "0.3" +criterion = "0.3.5" yatp = { git = "https://github.com/tikv/yatp.git" } [target.'cfg(not(target_env = "msvc"))'.dev-dependencies] -tikv-jemallocator = "0.4.0" +tikv-jemallocator = "0.5.0" [[bench]] name = "bench" diff --git a/skiplist/benches/bench.rs b/skiplist/benches/bench.rs index affde9dd..374da06a 100644 --- a/skiplist/benches/bench.rs +++ b/skiplist/benches/bench.rs @@ -53,13 +53,13 @@ fn bench_read_write_skiplist_frac(b: &mut Bencher<'_>, frac: &usize) { let mut rng = rand::thread_rng(); while !s.load(Ordering::SeqCst) { let key = random_key(&mut rng); - let case = (key, frac > rng.gen_range(0, 11)); + let case = (key, frac > rng.gen_range(0..11)); skiplist_round(&l, &case, &v); } }); let mut rng = rand::thread_rng(); b.iter_batched_ref( - || (random_key(&mut rng), frac > rng.gen_range(0, 11)), + || (random_key(&mut rng), frac > rng.gen_range(0..11)), |case| skiplist_round(&list, case, &value), BatchSize::SmallInput, ); @@ -104,7 +104,7 @@ fn bench_read_write_map_frac(b: &mut Bencher<'_>, frac: &usize) { let h = thread::spawn(move || { let mut rng = rand::thread_rng(); while !s.load(Ordering::SeqCst) { - let f = rng.gen_range(0, 11); + let f = rng.gen_range(0..11); let case = (random_key(&mut rng), f < frac); map_round(&m, &case, &v); } @@ -112,7 +112,7 @@ fn bench_read_write_map_frac(b: &mut Bencher<'_>, frac: &usize) { let mut rng = rand::thread_rng(); b.iter_batched_ref( || { - let f = rng.gen_range(0, 11); + let f = rng.gen_range(0..11); (random_key(&mut rng), f < frac) }, |case| map_round(&map, case, &value), diff --git a/src/checksum.rs b/src/checksum.rs index b6126b82..98768845 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,6 +1,7 @@ -use crate::{Error, Result}; use proto::meta::{checksum::Algorithm as ChecksumAlgorithm, Checksum}; +use crate::{Error, Result}; + pub fn calculate_checksum(data: &[u8], algo: ChecksumAlgorithm) -> u64 { match algo { ChecksumAlgorithm::Crc32c => { diff --git a/src/db/opt.rs b/src/db/opt.rs index c785c3c9..edd3bef3 100644 --- a/src/db/opt.rs +++ b/src/db/opt.rs @@ -1,10 +1,10 @@ -use skiplist::MAX_NODE_SIZE; +use std::cmp; -use super::*; use getset::Setters; +use skiplist::MAX_NODE_SIZE; +use super::*; use crate::{entry::Entry, opt}; -use std::cmp; #[derive(Clone, Setters)] pub struct AgateOptions { diff --git a/src/lib.rs b/src/lib.rs index 7f01a66e..73f37d74 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,8 +29,8 @@ pub use iterator::IteratorOptions; pub use iterator_trait::AgateIterator; pub use opt::{ChecksumVerificationMode, Options as TableOptions}; pub use skiplist::Skiplist; -pub use table::merge_iterator::Iterators; -pub use table::ConcatIterator; -pub use table::MergeIterator; -pub use table::{builder::Builder as TableBuilder, Table}; +pub use table::{ + builder::Builder as TableBuilder, merge_iterator::Iterators, ConcatIterator, MergeIterator, + Table, +}; pub use value::Value; diff --git a/src/managed_db.rs b/src/managed_db.rs index 8fe4d1ce..b617ba47 100644 --- a/src/managed_db.rs +++ b/src/managed_db.rs @@ -1,9 +1,7 @@ -use crate::db::Agate; -use crate::ops::transaction::Transaction; -use crate::Result; - use std::sync::Arc; +use crate::{db::Agate, ops::transaction::Transaction, Result}; + impl crate::db::Core { /// Follows the same logic as `new_transaction`, but uses the provided read timestamp. pub fn new_transaction_at(self: &Arc, read_ts: u64, update: bool) -> Transaction { diff --git a/src/manifest.rs b/src/manifest.rs index 347489c1..6943e88a 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -7,7 +7,7 @@ use std::{ }; use bytes::{Buf, BufMut, BytesMut}; -use crc::crc32; +use crc::{Crc, CRC_32_ISCSI}; use prost::Message; use proto::meta::{ manifest_change::Operation as ManifestChangeOp, ManifestChange, ManifestChangeSet, @@ -15,6 +15,8 @@ use proto::meta::{ use crate::{util, AgateOptions, Error, Result}; +pub const CRC32_CASTAGNOLI: Crc = Crc::::new(&CRC_32_ISCSI); + pub const MANIFEST_FILENAME: &str = "MANIFEST"; const MANIFEST_REWRITE_FILENAME: &str = "MANIFEST_REWRITE"; const MANIFEST_DELETION_REWRITE_THRESHOLD: usize = 10000; @@ -134,7 +136,7 @@ impl Manifest { offset += length; - if crc32::checksum_castagnoli(&buf) != (&len_crc_buf[4..]).get_u32() { + if CRC32_CASTAGNOLI.checksum(&buf) != (&len_crc_buf[4..]).get_u32() { return Err(Error::CustomError("bad checksum".to_string())); } @@ -241,7 +243,7 @@ impl ManifestFile { let mut len_crc_buf = vec![0; 8]; (&mut len_crc_buf[..4]).put_u32(change_buf.len() as u32); - (&mut len_crc_buf[4..]).put_u32(crc32::checksum_castagnoli(&change_buf)); + (&mut len_crc_buf[4..]).put_u32(CRC32_CASTAGNOLI.checksum(&change_buf)); buf.extend_from_slice(&len_crc_buf); buf.extend_from_slice(&change_buf); @@ -294,7 +296,7 @@ impl ManifestFile { } else { let mut len_crc_buf = vec![0; 8]; (&mut len_crc_buf[..4]).put_u32(buf.len() as u32); - (&mut len_crc_buf[4..]).put_u32(crc32::checksum_castagnoli(&buf)); + (&mut len_crc_buf[4..]).put_u32(CRC32_CASTAGNOLI.checksum(&buf)); len_crc_buf.extend_from_slice(&buf); inner.file.as_mut().unwrap().write_all(&len_crc_buf)?; diff --git a/src/ops/transaction.rs b/src/ops/transaction.rs index 2fefaa3a..a37b1016 100644 --- a/src/ops/transaction.rs +++ b/src/ops/transaction.rs @@ -87,11 +87,7 @@ impl Transaction { let mut entries: Vec<_> = self.pending_writes.values().cloned().collect(); entries.sort_by(|x, y| { let cmp = COMPARATOR.compare_key(&x.key, &y.key); - if reversed { - cmp.reverse() - } else { - cmp - } + if reversed { cmp.reverse() } else { cmp } }); Some(PendingWritesIterator::new(self.read_ts, reversed, entries)) @@ -458,7 +454,7 @@ impl AgateIterator for PendingWritesIterator { let key = user_key(key); self.next_idx = crate::util::search(self.entries.len(), |idx| { // Should not use COMPARATOR when compare without ts. - let cmp = (&self.entries[idx].key[..]).cmp(key); + let cmp = self.entries[idx].key[..].cmp(key); if !self.reversed { cmp != Less } else { diff --git a/src/ops/transaction_test.rs b/src/ops/transaction_test.rs index 01869769..5c89edbb 100644 --- a/src/ops/transaction_test.rs +++ b/src/ops/transaction_test.rs @@ -1,17 +1,15 @@ -use crate::Error; - -use crate::assert_bytes_eq; +use crate::{assert_bytes_eq, Error}; /// Tests in managed mode. mod managed_db { + use bytes::{Bytes, BytesMut}; + + use super::*; use crate::{ db::tests::{generate_test_agate_options, run_agate_test, with_payload}, entry::Entry, AgateOptions, }; - use bytes::{Bytes, BytesMut}; - - use super::*; fn default_test_managed_opts() -> AgateOptions { let mut opts = generate_test_agate_options(); @@ -136,6 +134,12 @@ mod normal_db { Arc, }; + use bytes::{Bytes, BytesMut}; + use crossbeam_channel::select; + use rand::Rng; + use yatp::{task::callback::Handle, Builder}; + + use super::*; use crate::{ closer::Closer, db::tests::{generate_test_agate_options, run_agate_test}, @@ -146,12 +150,6 @@ mod normal_db { value::VALUE_DELETE, Agate, AgateOptions, }; - use bytes::{Bytes, BytesMut}; - use crossbeam_channel::select; - use rand::Rng; - use yatp::{task::callback::Handle, Builder}; - - use super::*; #[test] fn test_txn_simple() { @@ -259,7 +257,7 @@ mod normal_db { let agate = agatedb.clone(); pool.spawn(move |_: &mut Handle<'_>| { let mut txn = agate.new_transaction(true); - let delta = rand::thread_rng().gen_range(0, 100); + let delta = rand::thread_rng().gen_range(0..100); for i in 0..3 { txn.set_entry(Entry::new(key(i), Bytes::from((100 - delta).to_string()))) diff --git a/src/table/concat_iterator.rs b/src/table/concat_iterator.rs index d4743ffd..b736b1a2 100644 --- a/src/table/concat_iterator.rs +++ b/src/table/concat_iterator.rs @@ -153,7 +153,7 @@ mod tests { let mut rng = thread_rng(); for i in 0..20 { - let size = rng.gen_range(100, 200); + let size = rng.gen_range(100..200); let data: Vec<(Bytes, Bytes)> = (cnt..cnt + size) .map(|x| { ( diff --git a/src/table/merge_iterator.rs b/src/table/merge_iterator.rs index 612398f4..dd08460d 100644 --- a/src/table/merge_iterator.rs +++ b/src/table/merge_iterator.rs @@ -235,8 +235,10 @@ impl AgateIterator for MergeIterator { #[cfg(test)] mod tests { use super::*; - use crate::assert_bytes_eq; - use crate::format::{key_with_ts, user_key}; + use crate::{ + assert_bytes_eq, + format::{key_with_ts, user_key}, + }; pub struct VecIterator { vec: Vec,