Use FetchContent instead of ExternalProject and git submodules in CMa…

…ke (#564) Currently, we are using git submodules to manage external libraries which need to pull full git histories. CMake gives a light way `FetchContent` to handle this scenario after 3.11, we can only fetch the specify tag or commit instead of full histories. Another minor change was that the CMake minimal required will be 3.13 to support the FetchContent instruction. Co-authored-by: tison <wander4096@gmail.com> Co-authored-by: hulk <hulk.website@gmail.com>
apache · May 11, 2022 · a51be76 · a51be76
1 parent b1e6a44
commit a51be76
Show file tree

Hide file tree

Showing 13 changed files with 237 additions and 350 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.13)
 project(kvrocks
         VERSION 999.999.999
         DESCRIPTION "NoSQL which based on rocksdb and compatible with the Redis protocol"
@@ -40,30 +40,24 @@ endif ()
 
 if(NOT DISABLE_JEMALLOC)
     include(cmake/jemalloc.cmake)
-    list(APPEND EXTERNAL_LIBS PRIVATE ${jemalloc_LIBRARIES})
-    list(APPEND EXTERNAL_INCS PRIVATE ${jemalloc_INCLUDE_DIRS})
+    list(APPEND EXTERNAL_LIBS PRIVATE jemalloc)
 endif()
 
+include(cmake/gtest.cmake)
 include(cmake/glog.cmake)
 include(cmake/snappy.cmake)
 include(cmake/rocksdb.cmake)
 include(cmake/libevent.cmake)
 include(cmake/lua.cmake)
 
-list(APPEND EXTERNAL_LIBS PRIVATE ${glog_LIBRARIES})
-list(APPEND EXTERNAL_INCS PRIVATE ${glog_INCLUDE_DIRS})
+list(APPEND EXTERNAL_LIBS PRIVATE glog)
+list(APPEND EXTERNAL_LIBS PRIVATE snappy)
 
-list(APPEND EXTERNAL_LIBS PRIVATE ${rocksdb_LIBRARIES})
-list(APPEND EXTERNAL_INCS PRIVATE ${rocksdb_INCLUDE_DIRS})
+list(APPEND EXTERNAL_LIBS PRIVATE rocksdb_with_headers)
 
-list(APPEND EXTERNAL_LIBS PRIVATE ${snappy_LIBRARIES})
-list(APPEND EXTERNAL_INCS PRIVATE ${snappy_INCLUDE_DIRS})
+list(APPEND EXTERNAL_LIBS PRIVATE event_with_headers)
 
-list(APPEND EXTERNAL_LIBS PRIVATE ${libevent_LIBRARIES})
-list(APPEND EXTERNAL_INCS PRIVATE ${libevent_INCLUDE_DIRS})
-
-list(APPEND EXTERNAL_LIBS PRIVATE ${lua_LIBRARIES})
-list(APPEND EXTERNAL_INCS PRIVATE ${lua_INCLUDE_DIRS})
+list(APPEND EXTERNAL_LIBS PRIVATE lua)
 
 # End dependencies
 
@@ -88,13 +82,8 @@ if(ENBALE_ASAN)
     target_compile_options(kvrocks PRIVATE -fsanitize=address)
     target_link_libraries(kvrocks PRIVATE -fsanitize=address)
 endif()
-add_dependencies(kvrocks libevent glog snappy rocksdb lua)
-if(NOT DISABLE_JEMALLOC)
-    add_dependencies(kvrocks jemalloc)
-endif()
 
 target_include_directories(kvrocks PRIVATE ${PROJECT_BINARY_DIR})
-target_include_directories(kvrocks ${EXTERNAL_INCS})
 find_package(Threads REQUIRED)
 if(THREADS_HAVE_PTHREAD_ARG)
     target_compile_options(kvrocks PUBLIC "-pthread")
@@ -203,9 +192,7 @@ if(ENBALE_ASAN)
     target_compile_options(kvrocks2redis PRIVATE -fno-omit-frame-pointer -fsanitize=address)
     target_link_libraries(kvrocks2redis PRIVATE -fno-omit-frame-pointer -fsanitize=address)
 endif()
-add_dependencies(kvrocks2redis libevent glog rocksdb)
 target_include_directories(kvrocks2redis PRIVATE ${PROJECT_BINARY_DIR})
-target_include_directories(kvrocks2redis ${EXTERNAL_INCS})
 if(UNWIND_LIB)
     target_link_libraries(kvrocks2redis PRIVATE -lunwind)
 endif()
@@ -404,13 +391,10 @@ add_executable(unittest
         src/rand.cc
         src/rand.h)
 
-add_dependencies(unittest glog rocksdb snappy lua)
 target_compile_features(unittest PRIVATE cxx_std_11)
-message(${snappy_LIBRARIES})
-target_link_libraries(unittest PRIVATE ${EXTERNAL_LIBS} -lgtest)
+target_link_libraries(unittest PRIVATE ${EXTERNAL_LIBS} gtest_main)
 if(UNWIND_LIB)
     target_link_libraries(unittest PRIVATE -lunwind)
 endif()
 target_include_directories(unittest PRIVATE ${PROJECT_BINARY_DIR})
-target_include_directories(unittest ${EXTERNAL_INCS})
 target_include_directories(unittest PRIVATE src)
diff --git a/build.sh b/build.sh
@@ -57,12 +57,11 @@ if [ "$(printf '%s\n' "$CMAKE_REQUIRE_VERSION" "$CMAKE_VERSION" | sort -V | head
     fi
     mkdir -p $BUILD_DIR/cmake
     cd $BUILD_DIR
-    curl -O -L https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz
-    tar -zxf cmake-3.13.2.tar.gz && cd cmake-3.13.2
+    CMAKE_DOWNLOAD_VERSION=3.23.1
+    curl -O -L https://github.com/Kitware/CMake/releases/download/v$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.tar.gz
+    tar -zxf cmake-$CMAKE_DOWNLOAD_VERSION.tar.gz && cd cmake-$CMAKE_DOWNLOAD_VERSION
     ./bootstrap --prefix=$CMAKE_INSTALL_DIR && make && make install && cd ../..
     CMAKE_BIN=$CMAKE_INSTALL_DIR/bin/cmake
 fi
 
-git submodule init
-git submodule update
 cd $BUILD_DIR && $CMAKE_BIN -DCMAKE_BUILD_TYPE=Release .. && make -j4 kvrocks kvrocks2redis
diff --git a/cmake/gflags.cmake b/cmake/gflags.cmake
diff --git a/cmake/glog.cmake b/cmake/glog.cmake
@@ -15,48 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-if (NOT __GLOG_INCLUDED)
-    set(__GLOG_INCLUDED TRUE)
-    find_package(Threads)
-    set(CMAKE_INSTALL_LIBDIR lib)
-    # build directory
-    set(glog_PREFIX ${CMAKE_BUILD_DIRECTORY}/external/glog-prefix)
-    # install directory
-    set(glog_INSTALL ${CMAKE_BUILD_DIRECTORY}/external/glog-install)
+include_guard()
 
-    if (UNIX)
-        set(GLOG_EXTRA_COMPILER_FLAGS "-fPIC")
-    endif()
+include(FetchContent)
 
-    set(GLOG_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GLOG_EXTRA_COMPILER_FLAGS}")
-    set(GLOG_C_FLAGS "${CMAKE_C_FLAGS} ${GLOG_EXTRA_COMPILER_FLAGS}")
+FetchContent_Declare(glog
+  GIT_REPOSITORY https://github.com/google/glog
+  GIT_TAG v0.4.0
+)
 
-    ExternalProject_Add(glog
-        PREFIX ${glog_PREFIX}
-        #GIT_REPOSITORY "https://github.com/google/glog"
-        #GIT_TAG "v0.3.5"
-        SOURCE_DIR ${PROJECT_SOURCE_DIR}/external/glog
-        INSTALL_DIR ${glog_INSTALL}
-        CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
-		   -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
-                   -DCMAKE_INSTALL_PREFIX=${glog_INSTALL}
-                   -DBUILD_SHARED_LIBS=OFF
-                   -DBUILD_STATIC_LIBS=ON
-                   -DBUILD_PACKAGING=OFF
-                   -DBUILD_TESTING=OFF
-                   -DBUILD_NC_TESTS=OFF
-                   -DBUILD_CONFIG_TESTS=OFF
-                   -DINSTALL_HEADERS=ON
-                   -DWITH_GFLAGS=OFF
-                   -DCMAKE_C_FLAGS=${GLOG_C_FLAGS}
-                   -DCMAKE_CXX_FLAGS=${GLOG_CXX_FLAGS}
-        LOG_DOWNLOAD 1
-        LOG_CONFIGURE 1
-        LOG_INSTALL 1
-        )
-
-    set(glog_FOUND TRUE)
-    set(glog_INCLUDE_DIRS ${glog_INSTALL}/include)
-    set(glog_LIBRARIES ${glog_INSTALL}/${CMAKE_INSTALL_LIBDIR}/libglog.a)
-endif()
+include(cmake/utils.cmake)
 
+FetchContent_MakeAvailableWithArgs(glog
+  WITH_GFLAGS=OFF
+  WITH_GTEST=OFF
+)
diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake
@@ -15,38 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-if (NOT __GTEST_INCLUDED) # guard against multiple includes
-    set(__GTEST_INCLUDED TRUE)
+include_guard()
 
-    # gtest will use pthreads if it's available in the system, so we must link with it
-    find_package(Threads)
+include(FetchContent)
 
-    # build directory
-    set(gtest_PREFIX ${CMAKE_BUILD_DIRECTORY}/external/gtest-prefix)
-    # install directory
-    set(gtest_INSTALL ${CMAKE_BUILD_DIRECTORY}/external/gtest-install)
+FetchContent_Declare(gtest
+  GIT_REPOSITORY https://github.com/google/googletest
+  GIT_TAG release-1.11.0
+)
 
-    if (UNIX)
-        set(GTEST_EXTRA_COMPILER_FLAGS "-fPIC")
-    endif()
+include(cmake/utils.cmake)
 
-    set(GTEST_CXX_FLAGS ${CMAKE_CXX_FLAGS} ${GTEST_EXTRA_COMPILER_FLAGS})
-    set(GTEST_C_FLAGS ${CMAKE_C_FLAGS} ${GTEST_EXTRA_COMPILER_FLAGS})
-
-    ExternalProject_Add(gtest
-        PREFIX ${gtest_PREFIX}
-        SOURCE_DIR ${PROJECT_SOURCE_DIR}/external/googletest
-        INSTALL_DIR ${gtest_INSTALL}
-        CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
-                   -DCMAKE_INSTALL_PREFIX=${gtest_INSTALL}
-                   -DCMAKE_C_FLAGS=${GTEST_C_FLAGS}
-                   -DCMAKE_CXX_FLAGS=${GTEST_CXX_FLAGS}
-                   -DBUILD_GMOCK=OFF
-        LOG_DOWNLOAD 1
-        LOG_INSTALL 1
-        )
-
-    set(gtest_FOUND TRUE)
-    set(gtest_INCLUDE_DIRS ${gtest_INSTALL}/include)
-    set(gtest_LIBRARIES ${gtest_INSTALL}/${CMAKE_INSTALL_LIBDIR}/libgtest.a ${CMAKE_THREAD_LIBS_INIT})
-endif()
+FetchContent_MakeAvailableWithArgs(gtest
+  BUILD_GMOCK=OFF
+  INSTALL_GTEST=OFF
+)
diff --git a/cmake/jemalloc.cmake b/cmake/jemalloc.cmake
@@ -15,39 +15,34 @@
 # specific language governing permissions and limitations
 # under the License.
 
-if (NOT __JEMALLOC_INCLUDED) # guard against multiple includes
-    set(__JEMALLOC_INCLUDED TRUE)
+include_guard()
 
-    find_package(Threads)
-    # build directory
-    set(jemalloc_PREFIX ${CMAKE_BUILD_DIRECTORY}/external/jemalloc-prefix)
-    # install directory
-    set(jemalloc_INSTALL ${CMAKE_BUILD_DIRECTORY}/external/jemalloc-install)
-    set(JEMALLOC_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/jemalloc")
+include(FetchContent)
 
-    if (UNIX)
-        set(JEMALLOC_EXTRA_COMPILER_FLAGS "-fPIC")
-    endif()
+FetchContent_Declare(jemalloc
+  GIT_REPOSITORY https://github.com/jemalloc/jemalloc
+  GIT_TAG 12cd13cd418512d9e7596921ccdb62e25a103f87
+)
 
-    set(JEMALLOC_CXX_FLAGS ${CMAKE_CXX_FLAGS} ${JEMALLOC_EXTRA_COMPILER_FLAGS})
-    set(JEMALLOC_C_FLAGS ${CMAKE_C_FLAGS} ${JEMALLOC_EXTRA_COMPILER_FLAGS})
+FetchContent_GetProperties(jemalloc)
+if(NOT jemalloc_POPULATED)
+  FetchContent_Populate(jemalloc)
 
-    ExternalProject_Add(jemalloc
-        SOURCE_DIR ${JEMALLOC_SOURCE_DIR}
-        PREFIX ${jemalloc_PREFIX}
-        INSTALL_DIR ${jemalloc_INSTALL}
-        CONFIGURE_COMMAND ${JEMALLOC_SOURCE_DIR}/configure --enable-autogen --disable-libdl --with-jemalloc-prefix=""
-            --prefix=${jemalloc_INSTALL}
-        BUILD_COMMAND make
-        INSTALL_COMMAND make dist COMMAND make install
-    )
-    ExternalProject_Add_Step(jemalloc autoconf
-        COMMAND autoconf
-        WORKING_DIRECTORY ${JEMALLOC_SOURCE_DIR}
-        COMMENT  "Jemalloc autoconf"
-        LOG 1
-    )
-    set(jemalloc_FOUND TRUE)
-    set(jemalloc_INCLUDE_DIRS ${jemalloc_INSTALL}/include)
-    set(jemalloc_LIBRARIES ${jemalloc_INSTALL}/lib/libjemalloc.a ${CMAKE_THREAD_LIBS_INIT})
+  execute_process(COMMAND autoconf
+    WORKING_DIRECTORY ${jemalloc_SOURCE_DIR}
+  )
+  execute_process(COMMAND ${jemalloc_SOURCE_DIR}/configure -C --enable-autogen --disable-libdl --with-jemalloc-prefix=""
+    WORKING_DIRECTORY ${jemalloc_BINARY_DIR}
+  )
+  add_custom_target(make_jemalloc 
+    COMMAND make
+    WORKING_DIRECTORY ${jemalloc_BINARY_DIR}
+  )
 endif()
+
+find_package(Threads REQUIRED)
+
+add_library(jemalloc INTERFACE)
+target_include_directories(jemalloc INTERFACE $<BUILD_INTERFACE:${jemalloc_BINARY_DIR}/include>)
+target_link_libraries(jemalloc INTERFACE $<BUILD_INTERFACE:${jemalloc_BINARY_DIR}/lib/libjemalloc.a> Threads::Threads)
+add_dependencies(jemalloc make_jemalloc)
diff --git a/cmake/libevent.cmake b/cmake/libevent.cmake
@@ -15,39 +15,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-if (NOT __LIBEVENT_INCLUDED)
-    set(__LIBEVENT_INCLUDED TRUE)
-    # build directory
-    set(libevent_PREFIX ${CMAKE_BUILD_DIRECTORY}/external/libevent-prefix)
-    # install directory
-    set(libevent_INSTALL ${CMAKE_BUILD_DIRECTORY}/external/libevent-install)
+include_guard()
 
-    if (UNIX)
-        set(LIBEVENT_EXTRA_COMPILER_FLAGS "-fPIC")
-    endif()
+include(FetchContent)
 
-    set(LIBEVENT_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBEVENT_EXTRA_COMPILER_FLAGS}")
-    set(LIBEVENT_C_FLAGS "${CMAKE_C_FLAGS} ${LIBEVENT_EXTRA_COMPILER_FLAGS}")
+FetchContent_Declare(libevent
+  GIT_REPOSITORY https://github.com/libevent/libevent
+  GIT_TAG release-2.1.11-stable
+)
 
-    ExternalProject_Add(libevent
-        PREFIX ${libevent_PREFIX}
-        SOURCE_DIR ${PROJECT_SOURCE_DIR}/external/libevent
-        INSTALL_DIR ${libevent_INSTALL}
-        CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-                   -DCMAKE_INSTALL_PREFIX=${libevent_INSTALL}
-                   -DCMAKE_C_FLAGS=${LIBEVENT_C_FLAGS}
-                   -DCMAKE_CXX_FLAGS=${LIBEVENT_CXX_FLAGS}
-                   -DEVENT__DISABLE_TESTS=ON
-                   -DEVENT__DISABLE_REGRESS=ON
-                   -DEVENT__DISABLE_SAMPLES=ON
-                   -DEVENT__DISABLE_OPENSSL=ON
-                   -DEVENT__LIBRARY_TYPE=STATIC
-        LOG_DOWNLOAD 1
-        LOG_CONFIGURE 1
-        LOG_INSTALL 1
-        )
+include(cmake/utils.cmake)
 
-    set(libevent_FOUND TRUE)
-    set(libevent_INCLUDE_DIRS ${libevent_INSTALL}/include)
-    set(libevent_LIBRARIES ${libevent_INSTALL}/lib/libevent.a ${libevent_INSTALL}/lib/libevent_pthreads.a)
-endif()
+FetchContent_MakeAvailableWithArgs(libevent
+  EVENT__DISABLE_TESTS=ON
+  EVENT__DISABLE_REGRESS=ON
+  EVENT__DISABLE_SAMPLES=ON
+  EVENT__DISABLE_OPENSSL=ON
+  EVENT__LIBRARY_TYPE=STATIC
+  EVENT__DISABLE_BENCHMARK=ON
+)
+
+add_library(event_with_headers INTERFACE)
+target_include_directories(event_with_headers INTERFACE ${libevent_SOURCE_DIR}/include ${libevent_BINARY_DIR}/include)
+target_link_libraries(event_with_headers INTERFACE event event_pthreads)