speedup CI UT job (alibaba#1606)

* Split the original UT CI job into two separate jobs: one with SPL and one without SPL * fix: change design. Build .a and .so at the same CI UT job * fix * fix * fix * fix * fix * fix * fix
Abingcbc · Jul 18, 2024 · 8fc252e · 8fc252e
1 parent 087705c
commit 8fc252e
Show file tree

Hide file tree

Showing 44 changed files with 292 additions and 271 deletions.
diff --git a/.github/workflows/build-core-ut.yaml b/.github/workflows/build-core-ut.yaml
@@ -74,14 +74,15 @@ jobs:
           ENABLE_COMPATIBLE_MODE: OFF
           ENABLE_STATIC_LINK_CRT: ON
           WITHOUTGDB: ON
-          MAKE_JOBS: 8
+          MAKE_JOBS: 16
+          # BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
         run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)
 
       - name: Unit Test
         run: make unittest_core
 
       - name: Unit Test Coverage
-        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt"
+        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
 
       - name: Setup Python3.10
         uses: actions/setup-python@v5

diff --git a/.github/workflows/build-core.yaml b/.github/workflows/build-core.yaml
@@ -17,13 +17,13 @@ name: Build Core
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -37,7 +37,7 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ arc-runner-set-ilogtail ]
+        runner: [arc-runner-set-ilogtail]
       fail-fast: true
     steps:
       - name: prepare ubuntu environment
@@ -60,7 +60,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          
+
       - name: Check out code
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -17,13 +17,13 @@ name: Build
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -37,7 +37,7 @@ jobs:
       matrix:
         go-version: [1.19]
         # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
-        runner: [ arc-runner-set-ilogtail ]
+        runner: [arc-runner-set-ilogtail]
       fail-fast: true
     steps:
       - name: prepare ubuntu environment
@@ -51,7 +51,7 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           sudo -E apt-get -qq autoremove --purge
           sudo -E apt-get -qq clean
-          
+
       - name: Check disk space
         run: |
           df -hT $PWD
@@ -60,7 +60,7 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          
+
       - name: Check out code
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/e2e-framework.yaml b/.github/workflows/e2e-framework.yaml
@@ -17,13 +17,13 @@ name: E2E Core Test
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -34,8 +34,8 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        go-version: [ 1.19.10 ]
-        runner: [ ubuntu-latest ]
+        go-version: [1.19.10]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
       # Clean up space to prevent action from running out of disk space.
@@ -81,7 +81,7 @@ jobs:
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: [ CI ]
+    needs: [CI]
     steps:
       - name: Build Result
         run: echo "Just to make the GitHub merge button green"
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -17,13 +17,13 @@ name: E2E Test
 on:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - 'example_config/**'
-      - 'docker/**'
-      - 'k8s_template/**'
-      - 'changes/**'
-      - 'licenses/**'
-      - 'CHANGELOG.md'
+      - "docs/**"
+      - "example_config/**"
+      - "docker/**"
+      - "k8s_template/**"
+      - "changes/**"
+      - "licenses/**"
+      - "CHANGELOG.md"
   push:
     branches:
       - main
@@ -34,8 +34,8 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        go-version: [ 1.19.10 ]
-        runner: [ ubuntu-latest ]
+        go-version: [1.19.10]
+        runner: [ubuntu-latest]
       fail-fast: true
     steps:
       # Clean up space to prevent action from running out of disk space.
@@ -83,7 +83,7 @@ jobs:
   result:
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: [ CI ]
+    needs: [CI]
     steps:
       - name: Build Result
         run: echo "Just to make the GitHub merge button green"
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -25,14 +25,20 @@ endif ()
 
 # Options.
 option(BUILD_LOGTAIL "Build Logtail executable and tools" ON)
+# Used under the Android environment.
 option(BUILD_LOGTAIL_SHARED_LIBRARY "Build Logtail shared library")
 option(ENABLE_ENTERPRISE "enable enterprise feature")
 cmake_dependent_option(ENABLE_COMPATIBLE_MODE "Build Logtail in compatible mode (for low version Linux)" OFF "LINUX" OFF)
 cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT statically" OFF "LINUX" OFF)
 option(WITHOUTGDB "Build Logtail without gdb")
-option(WITHOUTSPL "Build Logtail and UT without SPL")
+option(WITHSPL "Build Logtail and UT with SPL" ON)
 option(BUILD_LOGTAIL_UT "Build unit test for Logtail")
 
+if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL)
+    message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.")
+    return()
+endif()
+
 if (ENABLE_ENTERPRISE)
     message(STATUS "Enable Enterprise Feature.")
     add_definitions(-D__ENTERPRISE__)
@@ -41,7 +47,7 @@ else ()
     include(${CMAKE_CURRENT_SOURCE_DIR}/options.cmake)
 endif ()
 
-if (WITHOUTSPL)
+if (NOT WITHSPL)
     add_definitions(-D__EXCLUDE_SPL__)
 endif()
 
@@ -92,7 +98,8 @@ endif ()
 include(${CMAKE_CURRENT_SOURCE_DIR}/utils.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/links.cmake)
-set(SUBDIR_SOURCE_FILES "")
+set(PLUGIN_SOURCE_FILES_CORE "")
+set(PLUGIN_SOURCE_FILES_SPL "")
 include(${CMAKE_CURRENT_SOURCE_DIR}/common/common.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/common/links.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/input/input.cmake)
@@ -127,7 +134,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 include_directories("/opt/logtail_spl/include")
 if (LINUX)
     include_directories(${CMAKE_CURRENT_SOURCE_DIR}/observer)
-    if (NOT WITHOUTSPL)
+    if (WITHSPL)
         include_directories(${CMAKE_CURRENT_SOURCE_DIR}/spl)
     endif()
 endif()
@@ -138,37 +145,39 @@ endforeach (DIR_NAME)
 
 foreach (DIR_NAME ${SUB_DIRECTORIES_LIST})
     file(GLOB TEMP_SOURCE_FILES ${DIR_NAME}/*.c ${DIR_NAME}/*.cc ${DIR_NAME}/*.cpp)
-    list(APPEND SOURCE_FILES_LIST ${TEMP_SOURCE_FILES})
+    list(APPEND FRAMEWORK_SOURCE_FILES ${TEMP_SOURCE_FILES})
 endforeach (DIR_NAME)
 
 if (ENABLE_ENTERPRISE)
     # remove several files in shennong/sdk
-    list(REMOVE_ITEM SOURCE_FILES_LIST ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp)
+    list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp)
 endif()
 
 # remove several files in go_pipeline
-list(REMOVE_ITEM SOURCE_FILES_LIST ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h)
+list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h)
 
 if(MSVC)
     # remove linux event listener
     file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Linux.cpp event_listener/*_Linux.h)
-    list(REMOVE_ITEM SOURCE_FILES_LIST ${REMOVE_EVENT_LISTENER_SOURCES})
-    set(WINDOWS_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
+    list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
 elseif(UNIX)
     # remove windows event listener
     file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Windows.cpp event_listener/*_Windows.h)
-    list(REMOVE_ITEM SOURCE_FILES_LIST ${REMOVE_EVENT_LISTENER_SOURCES})
+    list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
     if (LINUX)
         # observer
         file(GLOB_RECURSE APPEND_OBSERVER_SOURCES observer/*)
-        list(APPEND SOURCE_FILES_LIST ${APPEND_OBSERVER_SOURCES})
+        list(APPEND FRAMEWORK_SOURCE_FILES ${APPEND_OBSERVER_SOURCES})
+        if (WITHSPL)
+            set(SRC_FILES ${PLUGIN_SOURCE_FILES_SPL})
+        endif()
     endif()
-    set(UNIX_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
 endif()
+set(SRC_FILES ${SRC_FILES} ${FRAMEWORK_SOURCE_FILES} ${PLUGIN_SOURCE_FILES_CORE})
 
 # Generate SPL library.
 if (LINUX)
-    if (NOT WITHOUTSPL)
+    if (WITHSPL)
         add_subdirectory(spl)
     endif()
 endif()
@@ -177,47 +186,35 @@ endif()
 if (BUILD_LOGTAIL)
     if (ENABLE_ENTERPRISE)
         if (UNIX)
-            add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
+            add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${SRC_FILES})
         elseif (MSVC)
-            add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
+            add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${SRC_FILES})
         endif ()
     else ()
         if (UNIX)
-            add_executable(${LOGTAIL_TARGET} logtail.cpp ${UNIX_SOURCE_FILES})
+            add_executable(${LOGTAIL_TARGET} logtail.cpp ${SRC_FILES})
         elseif (MSVC)
-            add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
+            add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${SRC_FILES})
         endif ()
     endif()
 endif()
 
-if (BUILD_LOGTAIL_SHARED_LIBRARY)
-    if (NOT WITHOUTSPL)
-        message(ERROR, "generating logtail shared library is not supported with SPL.")
-    else()
-        if (ENABLE_ENTERPRISE)
-            if (UNIX)
-                add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
-            elseif (MSVC)
-                add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
-            endif ()
-        else ()
-            if (UNIX)
-                add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${UNIX_SOURCE_FILES})
-            elseif (MSVC)
-                add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
-            endif ()
-        endif()
+if (BUILD_LOGTAIL_SHARED_LIBRARY)  
+    if (ENABLE_ENTERPRISE)
+        if (UNIX)
+            add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${SRC_FILES})
+        elseif (MSVC)
+            add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${SRC_FILES})
+        endif ()
+    else ()
+        if (UNIX)
+            add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${SRC_FILES})
+        elseif (MSVC)
+            add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${SRC_FILES})
+        endif ()
     endif()
 endif ()
 
-# Logtail UT.
-if (BUILD_LOGTAIL_UT)
-    message(STATUS "Build unittest.")
-    include(CTest)
-    enable_testing()
-    add_subdirectory(unittest)
-endif ()
-
 # Generate independent libraries.
 # add_subdirectory(helper)
 add_subdirectory(go_pipeline)
@@ -226,8 +223,16 @@ add_subdirectory(common)
 # Link libraries.
 if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY)
     input_link(${LOGTAIL_TARGET})
-    processor_link(${LOGTAIL_TARGET})
+    processor_link(${LOGTAIL_TARGET} ${WITHSPL})
     flusher_link(${LOGTAIL_TARGET})
     all_link(${LOGTAIL_TARGET})
     common_link(${LOGTAIL_TARGET})
-endif()
+endif()
+
+# Logtail UT.
+if (BUILD_LOGTAIL_UT)
+    message(STATUS "Build unittest.")
+    include(CTest)
+    enable_testing()
+    add_subdirectory(unittest)
+endif ()
diff --git a/core/common/common.cmake b/core/common/common.cmake
@@ -45,4 +45,4 @@ elseif(UNIX)
 endif()
 
 # Set source files to parent
-set(SUBDIR_SOURCE_FILES ${SUBDIR_SOURCE_FILES} ${THIS_SOURCE_FILES_LIST})
+set(PLUGIN_SOURCE_FILES_CORE ${PLUGIN_SOURCE_FILES_CORE} ${THIS_SOURCE_FILES_LIST})