From f98441a9dc883d5c39c38c64f45c504c9413f8ef Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Wed, 7 Jun 2023 20:42:56 +0800 Subject: [PATCH 1/6] [cmake] rm leveldb sub --- .gitmodules | 4 ---- cmake/external/leveldb.cmake | 10 ++++++++-- third_party/leveldb | 1 - 3 files changed, 8 insertions(+), 7 deletions(-) delete mode 160000 third_party/leveldb diff --git a/.gitmodules b/.gitmodules index 70cef6c956939..7574183fbe090 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,10 +30,6 @@ path = third_party/xxhash url = https://github.com/Cyan4973/xxHash.git ignore = dirty -[submodule "third_party/leveldb"] - path = third_party/leveldb - url = https://github.com/google/leveldb - ignore = dirty [submodule "third_party/threadpool"] path = third_party/threadpool url = https://github.com/progschj/ThreadPool.git diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 4eb630ad0ce81..829c2d4df6e2d 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -14,6 +14,8 @@ include(ExternalProject) +set(LEVELDB_TAG v1.18) +set(LEVELDB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/leveldb) set(LEVELDB_PREFIX_DIR ${THIRD_PARTY_PATH}/leveldb) set(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb) set(LEVELDB_INCLUDE_DIR @@ -27,12 +29,16 @@ set(LEVELDN_CXXFLAGS "-fPIC") if(WITH_HETERPS AND WITH_PSLIB) set(LEVELDN_CXXFLAGS "${LEVELDN_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") endif() + +execute_process( + COMMAND ${GIT_EXECUTABLE} clone -b ${LEVELDB_TAG} + "https://github.com/google/leveldb" ${LEVELDB_SOURCE_DIR}) + ExternalProject_Add( extern_leveldb ${EXTERNAL_PROJECT_LOG_ARGS} PREFIX ${LEVELDB_PREFIX_DIR} - GIT_REPOSITORY "https://github.com/google/leveldb" - GIT_TAG v1.18 + SOURCE_DIR ${LEVELDB_SOURCE_DIR} UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND export "CXXFLAGS=${LEVELDN_CXXFLAGS}" && make -j diff --git a/third_party/leveldb b/third_party/leveldb deleted file mode 160000 index 803d69203a62f..0000000000000 --- a/third_party/leveldb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 803d69203a62faf50f1b77897310a3a1fcae712b From c910edc6cfd6633f57e346a9c71271c23d5191f7 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Wed, 7 Jun 2023 22:34:58 +0800 Subject: [PATCH 2/6] [cmake] add tag check --- cmake/external/leveldb.cmake | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 829c2d4df6e2d..923510d578a5e 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -30,9 +30,26 @@ if(WITH_HETERPS AND WITH_PSLIB) set(LEVELDN_CXXFLAGS "${LEVELDN_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") endif() -execute_process( - COMMAND ${GIT_EXECUTABLE} clone -b ${LEVELDB_TAG} - "https://github.com/google/leveldb" ${LEVELDB_SOURCE_DIR}) +file(GLOB LEVELDB_SOURCE_FILE_LIST ${LEVELDB_SOURCE_DIR}) +list(LENGTH LEVELDB_SOURCE_FILE_LIST RES_LEN) +if(RES_LEN EQUAL 0) + execute_process( + COMMAND ${GIT_EXECUTABLE} clone -b ${LEVELDB_TAG} + https://github.com/google/leveldb.git ${LEVELDB_SOURCE_DIR}) +else() + # check git tag + execute_process( + COMMAND ${GIT_EXECUTABLE} describe --abbrev=6 --always --tags + OUTPUT_VARIABLE VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + WORKING_DIRECTORY ${LEVELDB_SOURCE_DIR}) + if(NOT ${VERSION} STREQUAL ${LEVELDB_TAG}) + message( + WARNING "leveldb version is not ${VERSION}, checkout to ${LEVELDB_TAG}") + execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${LEVELDB_TAG} + WORKING_DIRECTORY ${LEVELDB_SOURCE_DIR}) + endif() +endif() ExternalProject_Add( extern_leveldb From f0fed748b94edc1d11ff2ac98467e08f92720c92 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Thu, 8 Jun 2023 00:44:20 +0800 Subject: [PATCH 3/6] [cmake] fix install dir --- cmake/external/leveldb.cmake | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 923510d578a5e..d43ac4604de24 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -56,16 +56,12 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} PREFIX ${LEVELDB_PREFIX_DIR} SOURCE_DIR ${LEVELDB_SOURCE_DIR} + INSTALL_DIR ${LEVELDB_INSTALL_DIR} UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND export "CXXFLAGS=${LEVELDN_CXXFLAGS}" && make -j ${NUM_OF_PROCESSOR} libleveldb.a - INSTALL_COMMAND - mkdir -p ${LEVELDB_INSTALL_DIR}/lib/ && cp - ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES} - && cp -r ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/include - ${LEVELDB_INSTALL_DIR}/ - BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES}) add_dependencies(extern_leveldb snappy) From 10ab78d16411eac86245f97b1d9e97ea2e697888 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Thu, 8 Jun 2023 13:48:39 +0800 Subject: [PATCH 4/6] [cmake] fix install error --- cmake/external/leveldb.cmake | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index d43ac4604de24..a488e9c954c2f 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -56,12 +56,15 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} PREFIX ${LEVELDB_PREFIX_DIR} SOURCE_DIR ${LEVELDB_SOURCE_DIR} - INSTALL_DIR ${LEVELDB_INSTALL_DIR} UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND export "CXXFLAGS=${LEVELDN_CXXFLAGS}" && make -j ${NUM_OF_PROCESSOR} libleveldb.a - INSTALL_COMMAND "" + INSTALL_COMMAND + mkdir -p ${LEVELDB_INSTALL_DIR}/lib/ && cp + ${LEVELDB_SOURCE_DIR}/libleveldb.a ${LEVELDB_LIBRARIES} && cp -r + ${LEVELDB_SOURCE_DIR}/include ${LEVELDB_INSTALL_DIR}/ + BUILD_IN_SOURCE 1 BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES}) add_dependencies(extern_leveldb snappy) From 66498ee6ae0b58763b6255296716408f8dc3c8c5 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Fri, 9 Jun 2023 13:22:01 +0800 Subject: [PATCH 5/6] [cmake] fix update error --- cmake/external/leveldb.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index a488e9c954c2f..84c86e3000773 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -38,6 +38,8 @@ if(RES_LEN EQUAL 0) https://github.com/google/leveldb.git ${LEVELDB_SOURCE_DIR}) else() # check git tag + file(REMOVE_RECURSE ${LEVELDB_PREFIX_DIR}) + file(REMOVE_RECURSE ${LEVELDB_INSTALL_DIR}) execute_process( COMMAND ${GIT_EXECUTABLE} describe --abbrev=6 --always --tags OUTPUT_VARIABLE VERSION From ee7e0462d203cfef4c6b7a0e7cdd76d7e83e59b0 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Fri, 9 Jun 2023 13:28:00 +0800 Subject: [PATCH 6/6] fix --- cmake/external/leveldb.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 84c86e3000773..a35b6f429fa30 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -38,14 +38,14 @@ if(RES_LEN EQUAL 0) https://github.com/google/leveldb.git ${LEVELDB_SOURCE_DIR}) else() # check git tag - file(REMOVE_RECURSE ${LEVELDB_PREFIX_DIR}) - file(REMOVE_RECURSE ${LEVELDB_INSTALL_DIR}) execute_process( COMMAND ${GIT_EXECUTABLE} describe --abbrev=6 --always --tags OUTPUT_VARIABLE VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET WORKING_DIRECTORY ${LEVELDB_SOURCE_DIR}) if(NOT ${VERSION} STREQUAL ${LEVELDB_TAG}) + file(REMOVE_RECURSE ${LEVELDB_PREFIX_DIR}) + file(REMOVE_RECURSE ${LEVELDB_INSTALL_DIR}) message( WARNING "leveldb version is not ${VERSION}, checkout to ${LEVELDB_TAG}") execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${LEVELDB_TAG}