Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rocksdb: add compression type zstd and lz4 #217

Merged
merged 12 commits into from
Dec 13, 2018
Merged

rocksdb: add compression type zstd and lz4 #217

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
638bf2d
rocksdb: add compression type zstd and lz4
Nov 19, 2018
0666d64
update rdsn
Nov 19, 2018
e334143
Merge branch 'master' of https://github.com/XiaoMi/pegasus
Nov 19, 2018
084871f
link zstd and lz4 to binaries
Nov 21, 2018
1ea5334
Merge branch 'master' of https://github.com/XiaoMi/pegasus
Nov 21, 2018
ebd7057
pack up zstd and lz4
Nov 21, 2018
2d8f81c
Merge branch 'master' into master
qinzuoyan Dec 3, 2018
30fea36
Merge branch 'master' of https://github.com/xiaomi/pegasus
Dec 12, 2018
b6e9e2f
fix travis
Dec 13, 2018
0ec8862
fix travis
Dec 13, 2018
9ad8729
fix travis
Dec 13, 2018
f64af67
fix travis
Dec 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ function run_build()

echo "INFO: start build rocksdb..."
ROCKSDB_BUILD_DIR="$ROOT/rocksdb/build"
CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER"
CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER -DWITH_LZ4=ON -DWITH_ZSTD=ON -DWITH_SNAPPY=ON -DWITH_BZ2=OFF"
if [ "$WARNING_ALL" == "YES" ]
then
echo "WARNING_ALL=YES"
Expand Down
3 changes: 2 additions & 1 deletion scripts/pack_server.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ copy_file `get_system_lib server snappy` ${pack}/bin
copy_file `get_system_lib server crypto` ${pack}/bin
copy_file `get_system_lib server ssl` ${pack}/bin
copy_file `get_system_lib server aio` ${pack}/bin
copy_file `get_system_lib server bz2` ${pack}/bin
copy_file `get_system_lib server zstd` ${pack}/bin
neverchanje marked this conversation as resolved.
Show resolved Hide resolved
copy_file `get_system_lib server lz4` ${pack}/bin

chmod +x ${pack}/bin/pegasus_* ${pack}/bin/*.sh
chmod -x ${pack}/bin/lib*
Expand Down
3 changes: 3 additions & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ link_directories(${PEGASUS_ROOT}/lib)
set(CMAKE_INSTALL_PREFIX ${PEGASUS_ROOT} CACHE STRING "" FORCE)
message (STATUS "pegasus Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX})

# rocksdb
set(ROCKSDB_LINK_LIBRARIES rocksdb lz4 zstd snappy)

add_subdirectory(base)
add_subdirectory(reporter)
add_subdirectory(base/test)
Expand Down
6 changes: 2 additions & 4 deletions src/geo/bench/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,8 @@ set(MY_PROJ_LIBS
s2
pegasus_client_static
fmt
rocksdb
z
bz2
snappy)
${ROCKSDB_LINK_LIBRARIES}
z)

set(MY_BOOST_PACKAGES system filesystem)

Expand Down
4 changes: 1 addition & 3 deletions src/server/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,8 @@ set(MY_PROJ_LIBS
PocoJSON
crypto
fmt
rocksdb
${ROCKSDB_LINK_LIBRARIES}
z
bz2
snappy
rt
aio
pthread)
Expand Down
7 changes: 5 additions & 2 deletions src/server/pegasus_server_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,15 +181,18 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
"pegasus.server",
"rocksdb_compression_type",
"snappy",
"rocksdb options.compression, default snappy. Supported: none, snappy.");
"rocksdb options.compression, default snappy. Supported: none, snappy, zstd, lz4.");
if (compression_str == "none") {
_db_opts.compression = rocksdb::kNoCompression;
} else if (compression_str == "snappy") {
_db_opts.compression = rocksdb::kSnappyCompression;
} else if (compression_str == "zstd") {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里仅仅增加这几行代码还不够吧，还要在依赖库的链接上把这两种压缩也加进来

_db_opts.compression = rocksdb::kZSTD;
} else if (compression_str == "lz4") {
_db_opts.compression = rocksdb::kLZ4Compression;
} else {
dassert("unsupported compression type: %s", compression_str.c_str());
}

if (_db_opts.compression != rocksdb::kNoCompression) {
// only compress levels >= 2
// refer to ColumnFamilyOptions::OptimizeLevelStyleCompaction()
Expand Down
4 changes: 2 additions & 2 deletions src/server/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ set(MY_PROJ_LIBS
PocoJSON
crypto
fmt
rocksdb
${ROCKSDB_LINK_LIBRARIES}
pegasus.base
gtest
z bz2 snappy rt aio pthread
z rt aio pthread
)
add_definitions(-DPEGASUS_UNIT_TEST)
add_definitions(-DENABLE_FAIL)
Expand Down
4 changes: 1 addition & 3 deletions src/shell/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,8 @@ set(MY_PROJ_LIBS
fmt
pegasus_geo_lib
s2
rocksdb
${ROCKSDB_LINK_LIBRARIES}
z
bz2
snappy
rt
aio
pthread)
Expand Down
4 changes: 1 addition & 3 deletions src/test/bench_test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ set(MY_PROJ_LIB_PATH "../../../rocksdb/build")

set(MY_PROJ_LIBS
pegasus_client_static
rocksdb
${ROCKSDB_LINK_LIBRARIES}
z
bz2
snappy
gflags)

set(MY_BOOST_PACKAGES system filesystem)
Expand Down