From 638bf2d27cdea60293c0ba81c4038920e45b772f Mon Sep 17 00:00:00 2001 From: neverchanje Date: Mon, 19 Nov 2018 17:32:29 +0800 Subject: [PATCH 1/8] rocksdb: add compression type zstd and lz4 --- src/server/pegasus_server_impl.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 26c0566482..7d166b17d9 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -181,11 +181,15 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) "pegasus.server", "rocksdb_compression_type", "snappy", - "rocksdb options.compression, default snappy. Supported: none, snappy."); + "rocksdb options.compression, default snappy. Supported: none, snappy, zstd, lz4."); if (compression_str == "none") { _db_opts.compression = rocksdb::kNoCompression; } else if (compression_str == "snappy") { _db_opts.compression = rocksdb::kSnappyCompression; + } else if (compression_str == "zstd") { + _db_opts.compression = rocksdb::kZSTD; + } else if (compression_str == "lz4") { + _db_opts.compression = rocksdb::kLZ4Compression; } else { dassert("unsupported compression type: %s", compression_str.c_str()); } From 0666d645917545f77ecd62c3b5d9c851591110dc Mon Sep 17 00:00:00 2001 From: neverchanje Date: Mon, 19 Nov 2018 17:37:32 +0800 Subject: [PATCH 2/8] update rdsn --- rdsn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdsn b/rdsn index bb2fb74bee..faa8bf06d4 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit bb2fb74bee731128e8ca88ceba1804cef967a16a +Subproject commit faa8bf06d422aca6de4dd5eef2e2c8644ed7f992 From 084871f2caa9caa6cdba3516b945a396ffd3c9e3 Mon Sep 17 00:00:00 2001 From: neverchanje Date: Wed, 21 Nov 2018 16:06:36 +0800 Subject: [PATCH 3/8] link zstd and lz4 to binaries --- run.sh | 2 +- src/CMakeLists.txt | 3 +++ src/geo/bench/CMakeLists.txt | 6 ++---- src/server/CMakeLists.txt | 4 +--- src/server/test/CMakeLists.txt | 4 ++-- src/shell/CMakeLists.txt | 4 +--- src/test/bench_test/CMakeLists.txt | 4 +--- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/run.sh b/run.sh index e83d4bad16..2eb337d4f8 100755 --- a/run.sh +++ b/run.sh @@ -200,7 +200,7 @@ function run_build() echo "INFO: start build rocksdb..." ROCKSDB_BUILD_DIR="$ROOT/rocksdb/build" - CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER" + CMAKE_OPTIONS="-DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER -DWITH_LZ4=ON -DWITH_ZSTD=ON -DWITH_SNAPPY=ON -DWITH_BZ2=OFF" if [ "$WARNING_ALL" == "YES" ] then echo "WARNING_ALL=YES" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6deae84ec1..e03110abce 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -41,6 +41,9 @@ link_directories(${PEGASUS_ROOT}/lib) set(CMAKE_INSTALL_PREFIX ${PEGASUS_ROOT} CACHE STRING "" FORCE) message (STATUS "pegasus Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX}) +# rocksdb +set(ROCKSDB_LINK_LIBRARIES rocksdb lz4 zstd snappy) + add_subdirectory(base) add_subdirectory(reporter) add_subdirectory(base/test) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 2963fa3463..cbea8d4a04 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -19,10 +19,8 @@ set(MY_PROJ_LIBS s2 pegasus_client_static fmt - rocksdb - z - bz2 - snappy) + ${ROCKSDB_LINK_LIBRARIES} + z) set(MY_BOOST_PACKAGES system filesystem) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index e9c22bf731..21b794e75f 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -36,10 +36,8 @@ set(MY_PROJ_LIBS PocoJSON crypto fmt - rocksdb + ${ROCKSDB_LINK_LIBRARIES} z - bz2 - snappy rt aio pthread) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 65fdc25cb1..d8f0c28f8d 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -32,10 +32,10 @@ set(MY_PROJ_LIBS PocoJSON crypto fmt - rocksdb + ${ROCKSDB_LINK_LIBRARIES} pegasus.base gtest - z bz2 snappy rt aio pthread + z rt aio pthread ) add_definitions(-DPEGASUS_UNIT_TEST) add_definitions(-DENABLE_FAIL) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 36143b60c5..c9ccf73f9c 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -37,10 +37,8 @@ set(MY_PROJ_LIBS fmt pegasus_geo_lib s2 - rocksdb + ${ROCKSDB_LINK_LIBRARIES} z - bz2 - snappy rt aio pthread) diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt index 54216d53ca..71610e440b 100644 --- a/src/test/bench_test/CMakeLists.txt +++ b/src/test/bench_test/CMakeLists.txt @@ -16,10 +16,8 @@ set(MY_PROJ_LIB_PATH "../../../rocksdb/build") set(MY_PROJ_LIBS pegasus_client_static - rocksdb + ${ROCKSDB_LINK_LIBRARIES} z - bz2 - snappy gflags) set(MY_BOOST_PACKAGES system filesystem) From ebd7057803ce8b2ffddf1b39c6591fe4bb4828a6 Mon Sep 17 00:00:00 2001 From: neverchanje Date: Wed, 21 Nov 2018 16:35:51 +0800 Subject: [PATCH 4/8] pack up zstd and lz4 --- scripts/pack_server.sh | 3 ++- src/server/pegasus_server_impl.cpp | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pack_server.sh b/scripts/pack_server.sh index 49523d9bbf..530b5d5505 100755 --- a/scripts/pack_server.sh +++ b/scripts/pack_server.sh @@ -102,7 +102,8 @@ copy_file `get_system_lib server snappy` ${pack}/bin copy_file `get_system_lib server crypto` ${pack}/bin copy_file `get_system_lib server ssl` ${pack}/bin copy_file `get_system_lib server aio` ${pack}/bin -copy_file `get_system_lib server bz2` ${pack}/bin +copy_file `get_system_lib server zstd` ${pack}/bin +copy_file `get_system_lib server lz4` ${pack}/bin chmod +x ${pack}/bin/pegasus_* ${pack}/bin/*.sh chmod -x ${pack}/bin/lib* diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index ab5b624714..48da6b03db 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -193,7 +193,6 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) } else { dassert("unsupported compression type: %s", compression_str.c_str()); } - if (_db_opts.compression != rocksdb::kNoCompression) { // only compress levels >= 2 // refer to ColumnFamilyOptions::OptimizeLevelStyleCompaction() From b6e9e2f9d65e2cea5a7827a56e63e2b2780e5bde Mon Sep 17 00:00:00 2001 From: neverchanje Date: Thu, 13 Dec 2018 15:21:35 +0800 Subject: [PATCH 5/8] fix travis --- .travis.yml | 13 ++++++++++--- scripts/pack_tools.sh | 3 ++- src/CMakeLists.txt | 6 +++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a475bc3802..b1468f1564 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,15 @@ os: linux compiler: - gcc +addons: + apt: + packages: + - libsnappy-dev + - liblz4-dev + cache: - ccache + - apt before_install: - wget https://raw.githubusercontent.com/xiaomi/pegasus-common/master/build-depends.tar.gz @@ -19,9 +26,9 @@ before_install: - cd .. install: - - sudo apt-get -f install - - sudo apt-get -y install libsnappy-dev - - sudo apt-get -y install libgflags-dev + - wget https://github.com/facebook/zstd/archive/v1.3.7.zip; unzip v1.3.7; cd zstd-1.3.7; + - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; make install -j8; + - cd .. before_script: - cd rdsn/thirdparty diff --git a/scripts/pack_tools.sh b/scripts/pack_tools.sh index 1988bedbf0..9f36f890d8 100755 --- a/scripts/pack_tools.sh +++ b/scripts/pack_tools.sh @@ -108,7 +108,8 @@ copy_file `get_system_lib shell snappy` ${pack}/DSN_ROOT/lib/ copy_file `get_system_lib shell crypto` ${pack}/DSN_ROOT/lib/ copy_file `get_system_lib shell ssl` ${pack}/DSN_ROOT/lib/ copy_file `get_system_lib shell aio` ${pack}/DSN_ROOT/lib/ -copy_file `get_system_lib shell bz2` ${pack}/DSN_ROOT/lib/ +copy_file `get_system_lib shell zstd` ${pack}/DSN_ROOT/lib/ +copy_file `get_system_lib shell lz4` ${pack}/DSN_ROOT/lib/ chmod -x ${pack}/DSN_ROOT/lib/* mkdir -p ${pack}/scripts diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e03110abce..2065b3ed75 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,7 +42,11 @@ set(CMAKE_INSTALL_PREFIX ${PEGASUS_ROOT} CACHE STRING "" FORCE) message (STATUS "pegasus Installation directory: CMAKE_INSTALL_PREFIX = " ${CMAKE_INSTALL_PREFIX}) # rocksdb -set(ROCKSDB_LINK_LIBRARIES rocksdb lz4 zstd snappy) +set(CMAKE_MODULE_PATH "${PEGASUS_PROJECT_DIR}/rocksdb/cmake/modules") +find_package(snappy REQUIRED) +find_package(zstd REQUIRED) +find_package(lz4 REQUIRED) +set(ROCKSDB_LINK_LIBRARIES rocksdb ${ZSTD_LIBRARIES} ${LZ4_LIBRARIES} ${SNAPPY_LIBRARIES}) add_subdirectory(base) add_subdirectory(reporter) From 0ec886201fab2f198bd051a3d190ed5b60d7af8b Mon Sep 17 00:00:00 2001 From: neverchanje Date: Thu, 13 Dec 2018 15:26:10 +0800 Subject: [PATCH 6/8] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b1468f1564..17a3b452e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ before_install: install: - wget https://github.com/facebook/zstd/archive/v1.3.7.zip; unzip v1.3.7; cd zstd-1.3.7; - - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; make install -j8; + - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=/usr/lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; make install -j8; - cd .. before_script: From 9ad87294157975ea28f97ba5b9b18d255b12daed Mon Sep 17 00:00:00 2001 From: neverchanje Date: Thu, 13 Dec 2018 15:37:24 +0800 Subject: [PATCH 7/8] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 17a3b452e2..37f7665586 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ before_install: install: - wget https://github.com/facebook/zstd/archive/v1.3.7.zip; unzip v1.3.7; cd zstd-1.3.7; - - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=/usr/lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; make install -j8; + - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; sudo make install -j8; - cd .. before_script: From f64af67d96d8af1fc5a4b108f485248b5b98c313 Mon Sep 17 00:00:00 2001 From: neverchanje Date: Thu, 13 Dec 2018 15:42:33 +0800 Subject: [PATCH 8/8] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 37f7665586..3be91295ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: install: - wget https://github.com/facebook/zstd/archive/v1.3.7.zip; unzip v1.3.7; cd zstd-1.3.7; - mkdir cmake-build; cd cmake-build; cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DZSTD_BUILD_PROGRAMS=OFF ../build/cmake; sudo make install -j8; - - cd .. + - cd ../.. before_script: - cd rdsn/thirdparty