diff --git a/.circleci/config.yml b/.circleci/config.yml index 370ad301a8dd..5ce10059bc59 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -877,60 +877,11 @@ jobs: - perform-benchmarks - post-benchmarks -workflows: +workflows: # Only jobs that haven't been successfully migrated to GitHub Actions version: 2 - jobs-linux-run-tests: - jobs: - - build-linux - - build-linux-cmake-with-folly - - build-linux-cmake-with-folly-lite-no-test - - build-linux-gcc-7-with-folly - - build-linux-gcc-7-with-folly-lite-no-test - - build-linux-cmake-with-folly-coroutines - - build-linux-cmake-with-benchmark - - build-linux-encrypted_env-no_compression - jobs-linux-run-tests-san: - jobs: - - build-linux-clang10-asan - - build-linux-clang10-ubsan - - build-linux-clang10-mini-tsan - - build-linux-static_lib-alt_namespace-status_checked - jobs-linux-no-test-run: - jobs: - - build-linux-release - - build-linux-release-rtti - - build-examples - - build-fuzzers - - build-linux-clang-no_test_run - - build-linux-clang-13-no_test_run - - build-linux-gcc-8-no_test_run - - build-linux-gcc-10-cxx20-no_test_run - - build-linux-gcc-11-no_test_run - - build-linux-arm-cmake-no_test_run - jobs-linux-other-checks: - jobs: - - build-linux-clang10-clang-analyze - - build-linux-unity-and-headers - - build-linux-mini-crashtest - jobs-windows: - jobs: - - build-windows-vs2019 - - build-cmake-mingw jobs-java: jobs: - - build-linux-java - build-linux-java-static - - build-macos-java - - build-macos-java-static - - build-macos-java-static-universal - - build-linux-java-pmd - jobs-macos: - jobs: - - build-macos - - build-macos-cmake: - run_even_tests: true - - build-macos-cmake: - run_even_tests: false jobs-linux-arm: jobs: - build-linux-arm diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml new file mode 100644 index 000000000000..909ca549b697 --- /dev/null +++ b/.github/actions/build-folly/action.yml @@ -0,0 +1,7 @@ +name: build-folly +runs: + using: composite + steps: + - name: Build folly and dependencies + run: make build_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml new file mode 100644 index 000000000000..da78fdae43e7 --- /dev/null +++ b/.github/actions/build-for-benchmarks/action.yml @@ -0,0 +1,8 @@ +name: build-for-benchmarks +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" + - name: Linux build for benchmarks + run: make V=1 J=8 -j8 release + shell: bash \ No newline at end of file diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml new file mode 100644 index 000000000000..0a217ff5516b --- /dev/null +++ b/.github/actions/increase-max-open-files-on-macos/action.yml @@ -0,0 +1,10 @@ +name: increase-max-open-files-on-macos +runs: + using: composite + steps: + - name: Increase max open files + run: |- + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml new file mode 100644 index 000000000000..4ad144740654 --- /dev/null +++ b/.github/actions/install-gflags-on-macos/action.yml @@ -0,0 +1,7 @@ +name: install-gflags-on-macos +runs: + using: composite + steps: + - name: Install gflags on macos + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml new file mode 100644 index 000000000000..b3695aed5da2 --- /dev/null +++ b/.github/actions/install-gflags/action.yml @@ -0,0 +1,7 @@ +name: install-gflags +runs: + using: composite + steps: + - name: Install gflags + run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + shell: bash \ No newline at end of file diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml new file mode 100644 index 000000000000..c6cd991ddc6b --- /dev/null +++ b/.github/actions/install-jdk8-on-macos/action.yml @@ -0,0 +1,9 @@ +name: install-jdk8-on-macos +runs: + using: composite + steps: + - name: Install JDK 8 on macos + run: |- + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml new file mode 100644 index 000000000000..0924f5058bb2 --- /dev/null +++ b/.github/actions/install-maven/action.yml @@ -0,0 +1,7 @@ +name: install-maven +runs: + using: composite + steps: + - name: Install maven + run: sudo apt-get update -y && sudo apt-get install -y maven + shell: bash \ No newline at end of file diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml new file mode 100644 index 000000000000..3eac7c7b3bed --- /dev/null +++ b/.github/actions/perform-benchmarks/action.yml @@ -0,0 +1,22 @@ +name: perform-benchmarks +runs: + using: composite + steps: + - name: Test low-variance benchmarks + run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000" + env: + LD_LIBRARY_PATH: "/usr/local/lib" + DURATION_RO: 300 + DURATION_RW: 500 + NUM_THREADS: 1 + MAX_BACKGROUND_JOBS: 4 + CI_TESTS_ONLY: 'true' + WRITE_BUFFER_SIZE_MB: 16 + TARGET_FILE_SIZE_BASE_MB: 16 + MAX_BYTES_FOR_LEVEL_BASE_MB: 64 + COMPRESSION_TYPE: none + CACHE_INDEX_AND_FILTER_BLOCKS: 1 + MIN_LEVEL_TO_COMPRESS: 3 + CACHE_SIZE_MB: 10240 + MB_WRITE_PER_SEC: 2 + shell: bash \ No newline at end of file diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml new file mode 100644 index 000000000000..9d9b07827d7d --- /dev/null +++ b/.github/actions/post-benchmarks/action.yml @@ -0,0 +1,14 @@ +name: post-benchmarks +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/benchmark-results" + - name: Send benchmark report to visualisation + run: |- + set +e + set +o pipefail + ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc + true + shell: bash \ No newline at end of file diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml new file mode 100644 index 000000000000..a9313895b9d1 --- /dev/null +++ b/.github/actions/post-pmd-steps/action.yml @@ -0,0 +1,10 @@ +name: post-pmd-steps +runs: + using: composite + steps: + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/pmd.xml" + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ github.workspace }}/java/target/site" \ No newline at end of file diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml new file mode 100644 index 000000000000..b41bfc308604 --- /dev/null +++ b/.github/actions/post-steps/action.yml @@ -0,0 +1,31 @@ +name: post-steps +runs: + using: composite + steps: + - uses: actions/checkout@v3 + - name: Setup tmate session #TEMPORARY! + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/test-results" + - uses: actions/upload-artifact@v3.1.3 + with: + path: LOG + - name: Compress Test Logs + run: tar -cvzf t.tar.gz t + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: t.tar.gz + - run: |- + mkdir -p ${{ runner.temp }}/core_dumps + cp core.* ${{ runner.temp }}/core_dumps + if: ${{ failure() }} + shell: bash + - uses: actions/upload-artifact@v3.1.3 + with: + path: "${{ runner.temp }}/core_dumps" diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml new file mode 100644 index 000000000000..2cc9195c884b --- /dev/null +++ b/.github/actions/pre-steps-macos/action.yml @@ -0,0 +1,5 @@ +name: pre-steps-macos +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" \ No newline at end of file diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml new file mode 100644 index 000000000000..b192c87ab923 --- /dev/null +++ b/.github/actions/pre-steps/action.yml @@ -0,0 +1,18 @@ +name: pre-steps +runs: + using: composite + steps: + - name: Setup Environment Variables + run: |- + echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV" + echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV" + echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV" + echo "GTEST_COLOR=1" >> "$GITHUB_ENV" + echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV" + echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV" + echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV" + echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV" + echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV" + echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV" + echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV" + shell: bash \ No newline at end of file diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml new file mode 100644 index 000000000000..dd19977e3aad --- /dev/null +++ b/.github/actions/setup-folly/action.yml @@ -0,0 +1,7 @@ +name: setup-folly +runs: + using: composite + steps: + - name: Checkout folly sources + run: make checkout_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml new file mode 100644 index 000000000000..0f76b12d0a89 --- /dev/null +++ b/.github/actions/windows-build-steps/action.yml @@ -0,0 +1,44 @@ +name: windows-build-steps +runs: + using: composite + steps: + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@v1.3.1 + - name: Custom steps + env: + THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty + CMAKE_HOME: C:/Program Files/CMake + CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe + CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe + JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8 + SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.1.8 + SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.1.8;${{ github.workspace }}/thirdparty/snappy-1.1.8/build + SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.1.8/build/Debug/snappy.lib + run: |- + echo ===================== Install Dependencies ===================== + choco install liberica8jdk -y + mkdir $Env:THIRDPARTY_HOME + cd $Env:THIRDPARTY_HOME + echo "Building Snappy dependency..." + curl -Lo snappy-1.1.8.zip https://github.com/google/snappy/archive/refs/tags/1.1.8.zip + unzip -q snappy-1.1.8.zip + cd snappy-1.1.8 + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" .. + msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + echo ======================== Build RocksDB ========================= + cd ${{ github.workspace }} + $env:Path = $env:JAVA_HOME + ";" + $env:Path + mkdir build + cd build + & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 .. + cd .. + echo "Building with VS version: $Env:CMAKE_GENERATOR" + msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64 + echo ========================= Test RocksDB ========================= + build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16 + echo ======================== Test RocksJava ======================== + cd build\java + & ctest -C Debug -j 16 + shell: pwsh diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml new file mode 100644 index 000000000000..1deb5fb19e53 --- /dev/null +++ b/.github/workflows/benchmark-linux.yml @@ -0,0 +1,10 @@ +name: facebook/rocksdb/benchmark-linux +on: [push, pull_request] # TEMPORARY: test on PR +jobs: + benchmark-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/build-for-benchmarks" + - uses: "./.github/actions/perform-benchmarks" + - uses: "./.github/actions/post-benchmarks" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000000..66784059a8e3 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,96 @@ +name: facebook/rocksdb/nightly +on: [push, pull_request] # TEMPORARY: test on PR +jobs: + build-format-compatible: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: test + run: |- + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + git config --global --add safe.directory /__w/rocksdb/rocksdb + tools/check_format_compatible.sh + - uses: "./.github/actions/post-steps" + build-linux-arm-test-full: + runs-on: + labels: arm64large + container: + image: ubuntu-2004:202111-02 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: make V=1 J=4 -j4 check + - uses: "./.github/actions/post-steps" + build-linux-run-microbench: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: DEBUG_LEVEL=0 make -j32 run_microbench + - uses: "./.github/actions/post-steps" + build-linux-non-shm: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + TEST_TMPDIR: "/tmp/rocksdb_test_tmp" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang-13-asan-ubsan-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-valgrind: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: PORTABLE=1 make V=1 -j32 valgrind_test + - uses: "./.github/actions/post-steps" + build-windows-vs2022-avx2: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: AVX2 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + build-windows-vs2022: + runs-on: windows-2022 + env: + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" diff --git a/.github/workflows/pr-jobs-candidate.yml b/.github/workflows/pr-jobs-candidate.yml new file mode 100644 index 000000000000..d945aef80171 --- /dev/null +++ b/.github/workflows/pr-jobs-candidate.yml @@ -0,0 +1,62 @@ +name: facebook/rocksdb/pr-jobs-candidate +on: [push, pull_request] +jobs: + # These jobs would be in pr-jobs but are failing or otherwise broken for + # some reason. + # ============================ Java Jobs ============================ # + build-linux-java-static: + runs-on: + labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:centos6_x64-be + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + - uses: "./.github/actions/post-steps" + # =========================== ARM Jobs ============================ # + build-linux-arm: + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some + - uses: "./.github/actions/post-steps" + build-linux-arm-cmake-no_test_run: + runs-on: + labels: arm64large # GitHub hosted ARM runners do not yet exist + env: + JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64" + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV + which java && java -version + which javac && javac -version + - name: Build with cmake + run: |- + mkdir build + cd build + cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. + make -j4 + - name: Build Java with cmake + run: |- + rm -rf build + mkdir build + cd build + cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 .. + make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" diff --git a/.github/workflows/pr-jobs.yml b/.github/workflows/pr-jobs.yml new file mode 100644 index 000000000000..4d4a02e3b688 --- /dev/null +++ b/.github/workflows/pr-jobs.yml @@ -0,0 +1,508 @@ +name: facebook/rocksdb/pr-jobs +on: [push, pull_request] +jobs: + # NOTE: multiple workflows would be recommended, but the current GHA UI in + # PRs doesn't make it clear when there's an overall error with a workflow, + # making it easy to overlook something broken. Grouping everything into one + # workflow minimizes the problem because it will be suspicious if there are + # no GHA results. + # ========================= Linux With Tests ======================== # + build-linux: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: make V=1 J=32 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-cmake-mingw: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + - name: Build cmake-mingw + run: |- + export PATH=$JAVA_HOME/bin:$PATH + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)" + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check + - uses: "./.github/actions/post-steps" + build-linux-gcc-7-with-folly-lite-no-test: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-folly-coroutines: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + env: + CC: gcc-10 + CXX: g++-10 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/setup-folly" + - uses: "./.github/actions/build-folly" + - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)" + - uses: "./.github/actions/post-steps" + build-linux-cmake-with-benchmark: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - uses: "./.github/actions/post-steps" + build-linux-encrypted_env-no_compression: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n" + - uses: "./.github/actions/post-steps" + # ======================== Linux No Test Runs ======================= # + build-linux-release: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: "./db_stress --version" + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: if ./db_stress --version; then false; else true; fi + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: if ./db_stress --version; then false; else true; fi + - uses: "./.github/actions/post-steps" + build-linux-release-rtti: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: "./db_stress --version" + - run: make clean + - run: apt-get remove -y libgflags-dev + - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench + - run: if ./db_stress --version; then false; else true; fi + build-examples: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build examples + run: make V=1 -j4 static_lib && cd examples && make V=1 -j4 + - uses: "./.github/actions/post-steps" + build-fuzzers: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Build rocksdb lib + run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib + - name: Build fuzzers + run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer + - uses: "./.github/actions/post-steps" + build-linux-clang-no_test_run: + runs-on: + labels: 8-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all + - uses: "./.github/actions/post-steps" + build-linux-clang-13-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + build-linux-gcc-8-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-10-cxx20-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - uses: "./.github/actions/post-steps" + build-linux-gcc-11-no_test_run: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench + - uses: "./.github/actions/post-steps" + # ======================== Linux Other Checks ======================= # + check-format-and-targets: + runs-on: ubuntu-latest + steps: + - name: Checkout feature branch + uses: actions/checkout@v4.1.0 + - name: Fetch from upstream + run: | + git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream + - name: Where am I + run: | + echo git status && git status + echo "git remote -v" && git remote -v + echo git branch && git branch + - name: Setup Python + uses: actions/setup-python@v1 + - name: Install Dependencies + run: python -m pip install --upgrade pip + - name: Install argparse + run: pip install argparse + - name: Download clang-format-diff.py + run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py + - name: Check format + run: VERBOSE_CHECK=1 make check-format + - name: Compare buckify output + run: make check-buck-targets + - name: Simple source code checks + run: make check-sources + build-linux-clang10-clang-analyze: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze + - uses: "./.github/actions/post-steps" + - name: compress test report + run: tar -cvzf scan_build_report.tar.gz scan_build_report + if: failure() + - uses: actions/upload-artifact@v3.1.3 + with: + path: scan_build_report.tar.gz + build-linux-unity-and-headers: + runs-on: + labels: 4-core-ubuntu + container: + image: gcc:latest + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - run: apt-get update -y && apt-get install -y libgflags-dev + - name: Unity build + run: make V=1 -j8 unity_test + - run: make V=1 -j8 -k check-headers + - uses: "./.github/actions/post-steps" + build-linux-mini-crashtest: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush + - uses: "./.github/actions/post-steps" + # ======================= Linux with Sanitizers ===================== # + build-linux-clang10-asan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-clang10-ubsan: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check + - uses: "./.github/actions/post-steps" + build-linux-clang10-mini-tsan: + runs-on: + labels: 32-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - uses: "./.github/actions/post-steps" + build-linux-static_lib-alt_namespace-status_checked: + runs-on: + labels: 16-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - uses: "./.github/actions/post-steps" + # ========================= MacOS with Tests ======================== # + build-macos: + runs-on: macos-13 + env: + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Run all tests + run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all + - uses: "./.github/actions/post-steps" + build-macos-cmake: + runs-on: macos-13 + strategy: + matrix: + run_even_tests: [true, false] + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: cmake generate project file + run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. + - name: Build tests + run: cd build && make V=1 -j16 + - name: Run even tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2 + if: ${{ matrix.run_even_tests }} + - name: Run odd tests + run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2 + if: ${{ ! matrix.run_even_tests }} + - uses: "./.github/actions/post-steps" + # ======================== Windows with Tests ======================= # + # NOTE: some windows jobs are in "nightly" to save resources + build-windows-vs2019: + runs-on: windows-2019 + env: + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PORTABLE: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/windows-build-steps" + # ============================ Java Jobs ============================ # + build-linux-java: + runs-on: + labels: 4-core-ubuntu + container: + image: zjay437/rocksdb:0.6 + options: --shm-size=16gb + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=8 -j8 jtest + - uses: "./.github/actions/post-steps" + build-macos-java: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: make V=1 J=16 -j16 jtest + - uses: "./.github/actions/post-steps" + build-macos-java-static: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava x86 and ARM Static Libraries + run: make V=1 J=16 -j16 rocksdbjavastaticosx + - uses: "./.github/actions/post-steps" + build-macos-java-static-universal: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Universal Binary Static Library + run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub + - uses: "./.github/actions/post-steps" + build-linux-java-pmd: + runs-on: + labels: 4-core-ubuntu + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/install-maven" + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: PMD RocksDBJava + run: make V=1 J=8 -j8 jpmd + - uses: "./.github/actions/post-pmd-steps" diff --git a/.github/workflows/sanity_check.yml b/.github/workflows/sanity_check.yml deleted file mode 100644 index efc9d99cf372..000000000000 --- a/.github/workflows/sanity_check.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Check buck targets and code format -on: [push, pull_request] -permissions: - contents: read - -jobs: - check: - name: Check TARGETS file and code format - runs-on: ubuntu-latest - steps: - - name: Checkout feature branch - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fetch from upstream - run: | - git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream - - - name: Where am I - run: | - echo git status && git status - echo "git remote -v" && git remote -v - echo git branch && git branch - - - name: Setup Python - uses: actions/setup-python@v1 - - - name: Install Dependencies - run: python -m pip install --upgrade pip - - - name: Install argparse - run: pip install argparse - - - name: Download clang-format-diff.py - run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py - - - name: Check format - run: VERBOSE_CHECK=1 make check-format - - - name: Compare buckify output - run: make check-buck-targets - - - name: Simple source code checks - run: make check-sources