Merge branch 'main' into task_version_update

y-scope · Oct 29, 2024 · 10ed8f9 · 10ed8f9
2 parents 9cbd1aa + b92a6c6
commit 10ed8f9
Show file tree

Hide file tree

Showing 123 changed files with 7,772 additions and 4,768 deletions.
diff --git a/.github/actions/clp-core-build/action.yaml b/.github/actions/clp-core-build/action.yaml
@@ -8,6 +8,9 @@ inputs:
   use_published_image:
     description: "Whether to use the published container image"
     required: true
+  use_shared_libs:
+    description: "Whether to build the core binaries by linking against shared libraries"
+    required: true
   upload_binaries:
     description: "Whether to upload the core binaries"
     required: true
@@ -18,10 +21,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - shell: "bash"
-      working-directory: "./components/core"
-      run: "./tools/scripts/deps-download/download-all.sh"
-
     - if: "inputs.use_published_image == 'false'"
       uses: "actions/download-artifact@v4"
       with:
@@ -46,13 +45,21 @@ runs:
         fi
       shell: "bash"
 
+    - run: "./tools/scripts/deps-download/init.sh"
+      shell: "bash"
+
     - run: >-
         docker run
         --user $(id -u):$(id -g)
-        --volume "$GITHUB_WORKSPACE/components/core":/mnt/clp
+        --volume "$GITHUB_WORKSPACE":/mnt/clp
         --workdir /mnt/clp
         ${{steps.get_image_props.outputs.qualified_image_name}}
-        /mnt/clp/tools/scripts/utils/build-and-run-unit-tests.sh . build
+        bash -c "task deps:core &&
+        python3 /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.py
+        ${{inputs.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+        --source-dir /mnt/clp/components/core
+        --build-dir /mnt/clp/components/core/build
+        --num-jobs $(getconf _NPROCESSORS_ONLN)"
       shell: "bash"
 
     - if: "inputs.upload_binaries == 'true'"

diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml
@@ -8,19 +8,25 @@ on:
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
   push:
     paths:
       - ".github/workflows/clp-core-build-macos.yaml"
       - "components/core/cmake/**"
       - "components/core/CMakeLists.txt"
       - "components/core/src/**"
       - "components/core/tests/**"
-      - "components/core/tools/scripts/lib_install/macos-12/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
       - "components/core/tools/scripts/deps-download/**"
-      - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+      - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
   workflow_dispatch:
 
 concurrency:
@@ -30,7 +36,11 @@ concurrency:
 
 jobs:
   build-macos:
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        runner: ["macos-13", "macos-14"]
+        use_shared_libs: [true, false]
+    runs-on: "${{matrix.runner}}"
     steps:
       - uses: "actions/checkout@v4"
         with:
@@ -40,20 +50,28 @@ jobs:
       - name: "Remove preinstalled binaries which conflict with brew's installs"
         run: |
           rm -f /usr/local/bin/2to3*
+          rm -f /usr/local/bin/go*
           rm -f /usr/local/bin/idle3*
           rm -f /usr/local/bin/pydoc3*
           rm -f /usr/local/bin/python3*
 
       - name: "Install dependencies"
-        run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh"
+        run: "./components/core/tools/scripts/lib_install/macos/install-all.sh"
 
-      - name: "Download source dependencies"
+      - run: "./tools/scripts/deps-download/init.sh"
+        shell: "bash"
+
+      - run: "task deps:core"
         shell: "bash"
-        working-directory: "./components/core"
-        run: "./tools/scripts/deps-download/download-all.sh"
 
       - name: "Build CLP-core and run unit tests"
         shell: "bash"
         working-directory: "./components/core"
         # NOTE: We omit the Stopwatch tests since GH's macOS runner is too slow
-        run: "./tools/scripts/utils/build-and-run-unit-tests.sh . build ~[Stopwatch]"
+        run: >-
+          python3 ./tools/scripts/utils/build-and-run-unit-tests.py
+          ${{matrix.use_shared_libs == 'true' && '--use-shared-libs' || ''}}
+          --source-dir .
+          --build-dir build
+          --num-jobs $(getconf _NPROCESSORS_ONLN)
+          --test-spec "~[Stopwatch]"
diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml
@@ -8,15 +8,21 @@ on:
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
   push:
     paths:
       - ".github/actions/clp-core-build/action.yaml"
       - ".github/actions/clp-core-build-containers/action.yaml"
       - ".github/workflows/clp-core-build.yaml"
       - ".gitmodules"
       - "components/core/**"
-      - "!components/core/tools/scripts/lib_install/macos-12/**"
+      - "deps-tasks.yml"
+      - "Taskfile.yml"
+      - "tools/scripts/deps-download/**"
+      - "!components/core/tools/scripts/lib_install/macos/**"
   workflow_dispatch:
 
 env:
@@ -30,7 +36,7 @@ jobs:
   filter-relevant-changes:
     runs-on: "ubuntu-latest"
     outputs:
-      centos74_image_changed: "${{steps.filter.outputs.centos74_image}}"
+      centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}"
       ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}"
       ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
       clp_changed: "${{steps.filter.outputs.clp}}"
@@ -57,12 +63,12 @@ jobs:
           # image (since it would be different from the published image).
           base: "main"
           filters: |
-            centos74_image:
+            centos_stream_9_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
-              - "components/core/tools/docker-images/clp-env-base-centos7.4/**"
-              - "components/core/tools/scripts/lib_install/centos7.4/**"
+              - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**"
+              - "components/core/tools/scripts/lib_install/centos-stream-9/**"
             ubuntu_focal_image:
               - ".github/actions/**"
               - ".github/workflows/clp-core-build.yaml"
@@ -83,11 +89,13 @@ jobs:
               - "components/core/CMakeLists.txt"
               - "components/core/src/**"
               - "components/core/tests/**"
-              - "components/core/tools/scripts/deps-download/**"
-              - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh"
+              - "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
+              - "deps-tasks.yml"
+              - "Taskfile.yml"
+              - "tools/scripts/deps-download/**"
 
-  centos74-deps-image:
-    if: "needs.filter-relevant-changes.outputs.centos74_image_changed == 'true'"
+  centos-stream-9-deps-image:
+    if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'"
     needs: "filter-relevant-changes"
     runs-on: "ubuntu-latest"
     steps:
@@ -101,7 +109,7 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build-containers"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
           docker_context: "components/core"
@@ -161,14 +169,19 @@ jobs:
             ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
           token: "${{secrets.GITHUB_TOKEN}}"
 
-  centos74-binaries:
+  centos-stream-9-binaries:
     # Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
     if: >-
       success()
       || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
     needs:
-      - "centos74-deps-image"
+      - "centos-stream-9-deps-image"
       - "filter-relevant-changes"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "centos-stream-9-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -181,11 +194,12 @@ jobs:
 
       - uses: "./.github/actions/clp-core-build"
         env:
-          OS_NAME: "centos7.4"
+          OS_NAME: "centos-stream-9"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
-            ${{needs.filter-relevant-changes.outputs.centos74_image_changed == 'false'
+            ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
           upload_binaries: "false"
 
@@ -197,6 +211,15 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-focal-deps-image"
+    strategy:
+      matrix:
+        include:
+          - use_shared_libs: true
+            upload_binaries: false
+          - use_shared_libs: false
+            upload_binaries: true
+    name: "ubuntu-focal-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -212,10 +235,11 @@ jobs:
           OS_NAME: "ubuntu-focal"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
-          upload_binaries: "true"
+          upload_binaries: "${{matrix.upload_binaries}}"
           binaries_artifact_name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
 
   ubuntu-jammy-binaries:
@@ -226,6 +250,11 @@ jobs:
     needs:
       - "filter-relevant-changes"
       - "ubuntu-jammy-deps-image"
+    strategy:
+      matrix:
+        use_shared_libs: [true, false]
+    name: "ubuntu-jammy-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins"
+    continue-on-error: true
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -241,6 +270,7 @@ jobs:
           OS_NAME: "ubuntu-jammy"
         with:
           image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_shared_libs: "${{matrix.use_shared_libs}}"
           use_published_image: >-
             ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
             || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}

diff --git a/README.md b/README.md
@@ -8,7 +8,8 @@ YScope's Compressed Log Processor (CLP) compresses your logs, and allows you to
 compressed logs without decompression. CLP supports both JSON logs and unstructured (i.e., free
 text) logs. It also supports real-time log compression within several logging libraries. CLP also
 includes purpose-built web interfaces for searching and viewing the compressed logs. To learn more
-about it, you can read our [paper][clp-paper].
+about it, read our [2021 paper][clp-paper-21] about handling unstructured logs and our 
+[2024 paper][clp-paper-24] on extending it to JSON logs.
 
 # Benchmarks
 
@@ -27,7 +28,7 @@ index-less design, so for a fair comparison, we disabled MongoDB and PostgreSQL'
 left them enabled, MongoDB and PostgreSQL's compression ratio would be worse. We didn't disable
 indexing for Elasticsearch or Splunk since these tools are fundamentally index-based (i.e., logs
 cannot be searched without indexes). More details about our experimental methodology can be found in
-the [CLP paper][clp-paper].
+the [2021 paper][clp-paper-21] and the [2024 paper][clp-paper-24].
 
 # System Overview
 
@@ -94,7 +95,8 @@ If you would like a feature or want to report a bug, please file an issue and we
 [clp-ffi-go]: https://github.com/y-scope/clp-ffi-go
 [clp-ffi-py]: https://github.com/y-scope/clp-ffi-py
 [clp-loglib-py]: https://github.com/y-scope/clp-loglib-py
-[clp-paper]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf
+[clp-paper-21]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf
+[clp-paper-24]: https://www.usenix.org/system/files/osdi24-wang-rui.pdf 
 [core]: http://docs.yscope.com/clp/main/dev-guide/components-core
 [core-container]: http://docs.yscope.com/clp/main/user-guide/core-container
 [datasets]: https://docs.yscope.com/clp/main/user-guide/resources-datasets