GitHub actions and workflows

.circleci/config.yml

@@ -879,58 +879,6 @@ jobs:
 
 workflows:
   version: 2
-  jobs-linux-run-tests:
-    jobs:
-      - build-linux
-      - build-linux-cmake-with-folly
-      - build-linux-cmake-with-folly-lite-no-test
-      - build-linux-gcc-7-with-folly
-      - build-linux-gcc-7-with-folly-lite-no-test
-      - build-linux-cmake-with-folly-coroutines
-      - build-linux-cmake-with-benchmark
-      - build-linux-encrypted_env-no_compression
-  jobs-linux-run-tests-san:
-    jobs:
-      - build-linux-clang10-asan
-      - build-linux-clang10-ubsan
-      - build-linux-clang10-mini-tsan
-      - build-linux-static_lib-alt_namespace-status_checked
-  jobs-linux-no-test-run:
-    jobs:
-      - build-linux-release
-      - build-linux-release-rtti
-      - build-examples
-      - build-fuzzers
-      - build-linux-clang-no_test_run
-      - build-linux-clang-13-no_test_run
-      - build-linux-gcc-8-no_test_run
-      - build-linux-gcc-10-cxx20-no_test_run
-      - build-linux-gcc-11-no_test_run
-      - build-linux-arm-cmake-no_test_run
-  jobs-linux-other-checks:
-    jobs:
-      - build-linux-clang10-clang-analyze
-      - build-linux-unity-and-headers
-      - build-linux-mini-crashtest
-  jobs-windows:
-    jobs:
-      - build-windows-vs2019
-      - build-cmake-mingw
-  jobs-java:
-    jobs:
-      - build-linux-java
-      - build-linux-java-static
-      - build-macos-java
-      - build-macos-java-static
-      - build-macos-java-static-universal
-      - build-linux-java-pmd
-  jobs-macos:
-    jobs:
-      - build-macos
-      - build-macos-cmake:
-          run_even_tests: true
-      - build-macos-cmake:
-          run_even_tests: false
   jobs-linux-arm:
     jobs:
       - build-linux-arm

.github/actions/build-folly/action.yml

@@ -0,0 +1,7 @@
+name: build-folly
+runs:
+  using: composite
+  steps:
+  - name: Build folly and dependencies
+    run: make build_folly
+    shell: bash

.github/actions/build-for-benchmarks/action.yml

@@ -0,0 +1,8 @@
+name: build-for-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: "./.github/actions/pre-steps"
+  - name: Linux build for benchmarks
+    run: make V=1 J=8 -j8 release
+    shell: bash

.github/actions/checkout/action.yml

@@ -0,0 +1,10 @@
+name: checkout with git cli
+runs:
+  using: composite
+  steps:
+  - name: Checkout
+    run: |-
+      sudo sysctl -w kern.maxfiles=1048576
+      sudo sysctl -w kern.maxfilesperproc=1048576
+      sudo launchctl limit maxfiles 1048576
+    shell: bash

.github/actions/increase-max-open-files-on-macos/action.yml

@@ -0,0 +1,10 @@
+name: increase-max-open-files-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Increase max open files
+    run: |-
+      sudo sysctl -w kern.maxfiles=1048576
+      sudo sysctl -w kern.maxfilesperproc=1048576
+      sudo launchctl limit maxfiles 1048576
+    shell: bash

.github/actions/install-gflags-on-macos/action.yml

@@ -0,0 +1,7 @@
+name: install-gflags-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install gflags on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
+    shell: bash

.github/actions/install-gflags/action.yml

@@ -0,0 +1,7 @@
+name: install-gflags
+runs:
+  using: composite
+  steps:
+  - name: Install gflags
+    run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+    shell: bash

.github/actions/install-jdk8-on-macos/action.yml

@@ -0,0 +1,9 @@
+name: install-jdk8-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install JDK 8 on macos
+    run: |-
+      HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+      HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
+    shell: bash

.github/actions/install-maven/action.yml

@@ -0,0 +1,7 @@
+name: install-maven
+runs:
+  using: composite
+  steps:
+  - name: Install maven
+    run: sudo apt-get update -y && sudo apt-get install -y maven
+    shell: bash

.github/actions/perform-benchmarks/action.yml

@@ -0,0 +1,22 @@
+name: perform-benchmarks
+runs:
+  using: composite
+  steps:
+  - name: Test low-variance benchmarks
+    run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000"
+    env:
+      LD_LIBRARY_PATH: "/usr/local/lib"
+      DURATION_RO: 300
+      DURATION_RW: 500
+      NUM_THREADS: 1
+      MAX_BACKGROUND_JOBS: 4
+      CI_TESTS_ONLY: 'true'
+      WRITE_BUFFER_SIZE_MB: 16
+      TARGET_FILE_SIZE_BASE_MB: 16
+      MAX_BYTES_FOR_LEVEL_BASE_MB: 64
+      COMPRESSION_TYPE: none
+      CACHE_INDEX_AND_FILTER_BLOCKS: 1
+      MIN_LEVEL_TO_COMPRESS: 3
+      CACHE_SIZE_MB: 10240
+      MB_WRITE_PER_SEC: 2
+    shell: bash

.github/actions/post-benchmarks/action.yml

@@ -0,0 +1,14 @@
+name: post-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: "${{ runner.temp }}/benchmark-results"
+  - name: Send benchmark report to visualisation
+    run: |-
+      set +e
+      set +o pipefail
+      ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
+      true
+    shell: bash

.github/actions/post-pmd-steps/action.yml

@@ -0,0 +1,10 @@
+name: post-pmd-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: "${{ github.workspace }}/java/target/pmd.xml"
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: "${{ github.workspace }}/java/target/site"

.github/actions/post-steps/action.yml

@@ -0,0 +1,31 @@
+name: post-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v3
+  - name: Setup tmate session #TEMPORARY!
+    if: ${{ failure() }}
+    uses: mxschmitt/action-tmate@v3
+    with:
+      limit-access-to-actor: true
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: "${{ runner.temp }}/test-results"
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: LOG
+  - name: Compress Test Logs
+    run: tar -cvzf t.tar.gz t
+    if: ${{ failure() }}
+    shell: bash
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: t.tar.gz
+  - run: |-
+      mkdir -p ${{ runner.temp }}/core_dumps
+      cp core.* ${{ runner.temp }}/core_dumps
+    if: ${{ failure() }}
+    shell: bash
+  - uses: actions/upload-artifact@v3.1.3
+    with:
+      path: "${{ runner.temp }}/core_dumps"

.github/actions/pre-steps-macos/action.yml

@@ -0,0 +1,5 @@
+name: pre-steps-macos
+runs:
+  using: composite
+  steps:
+  - uses: "./.github/actions/pre-steps"

.github/actions/pre-steps/action.yml

@@ -0,0 +1,18 @@
+name: pre-steps
+runs:
+  using: composite
+  steps:
+  - name: Setup Environment Variables
+    run: |-
+      echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV"
+      echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV"
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV"
+      echo "GTEST_COLOR=1" >> "$GITHUB_ENV"
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV"
+      echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV"
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV"
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV"
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV"
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV"
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV"
+    shell: bash

.github/actions/setup-folly/action.yml

@@ -0,0 +1,7 @@
+name: setup-folly
+runs:
+  using: composite
+  steps:
+  - name: Checkout folly sources
+    run: make checkout_folly
+    shell: bash

.github/actions/windows-build-steps/action.yml

@@ -0,0 +1,38 @@
+name: windows-build-steps
+runs:
+  using: composite
+  steps:
+  - name: Add msbuild to PATH
+    uses: microsoft/setup-msbuild@v1.3.1
+  - name: Install thirdparty dependencies
+    run: |-
+      choco install liberica8jdk -y
+      mkdir $Env:THIRDPARTY_HOME
+      cd $Env:THIRDPARTY_HOME
+      echo "Building Snappy dependency..."
+      curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip
+      unzip -q snappy-1.1.8.zip
+      cd snappy-1.1.8
+      mkdir build
+      cd build
+      & cmake -G "$Env:CMAKE_GENERATOR" ..
+      msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+    shell: pwsh
+  - name: Build RocksDB
+    run: |-
+      $env:Path = $env:JAVA_HOME + ";" + $env:Path
+      mkdir build
+      cd build
+      & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 ..
+      cd ..
+      echo "Building with VS version: $Env:CMAKE_GENERATOR"
+      msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+    shell: pwsh
+  - name: Test RocksDB
+    run: build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+    shell: pwsh
+  - name: Test RocksJava
+    run: |-
+      cd build\java
+      & ctest -C Debug -j 16
+    shell: pwsh

.github/workflows/benchmark-linux.yml

@@ -0,0 +1,13 @@
+name: facebook/rocksdb/benchmark-linux
+on:
+  schedule:
+  - cron: 0 * * * *
+  workflow_dispatch:
+jobs:
+  benchmark-linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/build-for-benchmarks"
+    - uses: "./.github/actions/perform-benchmarks"
+    - uses: "./.github/actions/post-benchmarks"