Merge pull request #27 from TileDB-Inc/kerl/ubuntu-24-04 #615
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly builds on linux | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/linux.yml' | |
| - 'scripts/**' | |
| schedule: | |
| - cron: "0 2 * * *" # Every night at 2 AM UTC (9 PM EST; 10 PM EDT) | |
| workflow_dispatch: | |
| inputs: | |
| LIBTILEDB_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB | |
| default: dev | |
| TILEDB_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Py | |
| default: dev | |
| TILEDB_R_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-R | |
| default: main | |
| TILEDB_VCF_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| TileDB-Inc/TileDB-VCF | |
| default: main | |
| TILEDB_SOMA_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout | |
| single-cell-data/TileDB-SOMA | |
| default: main | |
| TILEDB_MARIADB_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-MariaDB | |
| default: master | |
| TILEDB_GO_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Go | |
| default: master | |
| TILEDB_CLOUD_PY_REF: | |
| description: >- | |
| The Git reference (tag, branch, commit) to checkout TileDB-Inc/TileDB-Cloud-Py | |
| default: main | |
| jobs: | |
| libtiledb: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Clone TileDB | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB | |
| ref: ${{ github.event.inputs.LIBTILEDB_REF || 'dev' }} | |
| # Use recent release tarball on branches for quicker feedback | |
| - name: Download release tarball | |
| if: github.ref_name != 'main' | |
| run: wget https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz | |
| - name: Unpack release tarball | |
| if: github.ref_name != 'main' | |
| run: | | |
| mkdir install-libtiledb | |
| tar xzf tiledb-linux-x86_64-*.tar.gz -C install-libtiledb | |
| ls -R install-libtiledb/ | |
| - name: Patch tiledb.pc from release tarball | |
| if: github.ref_name != 'main' | |
| run: | | |
| new_prefix=prefix=\"$(pwd)/install-libtiledb\" | |
| echo $new_prefix | |
| sed -i "1 s+.*+$new_prefix+" install-libtiledb/lib/pkgconfig/tiledb.pc | |
| cat install-libtiledb/lib/pkgconfig/tiledb.pc | |
| - name: Configure TileDB | |
| if: github.ref_name == 'main' | |
| run: | | |
| cmake -B build-libtiledb \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledb/ \ | |
| -D TILEDB_WERROR=ON \ | |
| -D TILEDB_SERIALIZATION=ON \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D TILEDB_S3=ON \ | |
| -D VCPKG_TARGET_TRIPLET=x64-linux-release | |
| - name: Build TileDB | |
| if: github.ref_name == 'main' | |
| run: cmake --build build-libtiledb -j $(nproc) --config Release | |
| - name: Install TileDB | |
| if: github.ref_name == 'main' | |
| run: cmake --build build-libtiledb --config Release --target install-tiledb | |
| - name: Upload libtiledb as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: | | |
| install-libtiledb/* | |
| retention-days: 14 | |
| if-no-files-found: error | |
| # Unable to run the test suite with `TILEDB_S3=ON` (required for | |
| # tiledbvcf-py tests) without configuring S3 credentials | |
| # - name: Test TileDB | |
| # if: github.ref_name == 'main' | |
| # run: | | |
| # make -C build-libtiledb/tiledb tests -j $(nproc) | |
| # ./build-libtiledb/tiledb/test/tiledb_unit --vfs native | |
| tiledb-py: | |
| runs-on: ubuntu-24.04 | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| PKG_CONFIG_PATH: ${{ github.workspace }}/install-libtiledb/lib/pkgconfig | |
| steps: | |
| - name: Clone TileDB-Py | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-Py | |
| ref: ${{ github.event.inputs.TILEDB_PY_REF || 'dev' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Build TileDB-Py | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| export TILEDB_PATH=$(pwd)/install-libtiledb/ | |
| python -m pip install -Cskbuild.cmake.define.TILEDB_REMOVE_DEPRECATIONS=OFF -v --editable .[test] | |
| - name: Import TileDB-Py | |
| run: | | |
| python -c "import tiledb; print('successful import')" | |
| python -c "import tiledb; print(tiledb.libtiledb.version())" | |
| python -c "import tiledb; print(tiledb.version())" | |
| - name: Test TileDB-Py | |
| run: pytest -vv | |
| - name: Create a wheel | |
| run: | | |
| export TILEDB_PATH=$(pwd)/install-libtiledb/ | |
| python -m pip wheel -Cskbuild.cmake.define.TILEDB_REMOVE_DEPRECATIONS=OFF -v --wheel-dir=dist . | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| path: dist/tiledb-*-linux_x86_64.whl | |
| retention-days: 14 | |
| if-no-files-found: error | |
| tiledb-r: | |
| runs-on: ubuntu-24.04 | |
| needs: libtiledb | |
| env: | |
| _R_CHECK_FORCE_SUGGESTS_: "FALSE" | |
| steps: | |
| - name: Clone TileDB-R | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-R | |
| ref: ${{ github.event.inputs.TILEDB_R_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Bootstrap | |
| run: ./.github/r-ci.sh bootstrap | |
| - name: Dependencies | |
| run: ./.github/r-ci.sh install_all | |
| - name: Build | |
| run: R CMD build --no-manual --no-build-vignettes . | |
| - name: Install | |
| run: | | |
| R CMD INSTALL --configure-args=--with-tiledb=$(pwd)/install-libtiledb --build tiledb_*.tar.gz | |
| - name: Upload R binary tarball as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledb-r-linux | |
| path: tiledb_*_R_x86_64-pc-linux-gnu.tar.gz | |
| retention-days: 14 | |
| if-no-files-found: error | |
| - name: Test | |
| run: R -e 'tinytest::test_package("tiledb")' | |
| libtiledbvcf: | |
| runs-on: ubuntu-latest | |
| needs: libtiledb | |
| env: | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: ${{ github.event.inputs.TILEDB_VCF_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Setup | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes cmake | |
| - name: Configure libtiledbvcf | |
| run: | | |
| cmake -S libtiledbvcf -B build-libtiledbvcf \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbvcf/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D TILEDB_WERROR=OFF | |
| - name: Build libtiledbvcf | |
| run: cmake --build build-libtiledbvcf -j $(nproc) --config Release | |
| - name: Install libtiledbvcf | |
| run: | | |
| cmake --build build-libtiledbvcf --config Release --target install-libtiledbvcf | |
| ls -R install-libtiledbvcf/ | |
| - name: Upload libtiledbvcf as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: | | |
| install-libtiledbvcf/* | |
| retention-days: 14 | |
| if-no-files-found: error | |
| - name: libtiledbvcf version | |
| run: ./install-libtiledbvcf/bin/tiledbvcf version | |
| - name: Confirm linking | |
| run: ldd install-libtiledbvcf/lib/libtiledbvcf.so | |
| - name: Install bcftools (for tests) | |
| run: sudo apt-get install --yes bcftools | |
| - name: Test libtiledbvcf | |
| run: | | |
| make -j $(nproc) -C build-libtiledbvcf/libtiledbvcf tiledb_vcf_unit | |
| ./build-libtiledbvcf/libtiledbvcf/test/tiledb_vcf_unit | |
| # cli tests (require bcftools) | |
| # USAGE: run-cli-tests.sh <build-dir> <inputs-dir> | |
| libtiledbvcf/test/run-cli-tests.sh build-libtiledbvcf libtiledbvcf/test/inputs | |
| tiledbvcf-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb, libtiledbvcf, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbvcf/lib" | |
| steps: | |
| - name: Clone TileDB-VCF | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-VCF | |
| ref: ${{ github.event.inputs.TILEDB_VCF_REF || 'main' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbvcf | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: install-libtiledbvcf | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install --prefer-binary \ | |
| pyarrow==11 | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install --prefer-binary tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Build tiledbvcf-py | |
| run: | | |
| export LIBTILEDBVCF_PATH=$(pwd)/install-libtiledbvcf/lib/ | |
| export TileDB_DIR=$(pwd)/install-libtiledb/lib/cmake/TileDB | |
| python -m pip install -v apis/python[test] | |
| - name: Confirm linking | |
| run: | | |
| echo "Shared objects in build directory" | |
| find apis/python -name '*.so*' | |
| echo "Installed shared objects" | |
| find $pythonLocation/lib/python*/site-packages/tiledbvcf -name '*.so*' | |
| echo "Run ldd on installed shared objects" | |
| find $pythonLocation/lib/python*/site-packages/tiledbvcf -name '*.so*' | xargs ldd | |
| echo "Check RPATH of installed shared object" | |
| readelf -d $pythonLocation/lib/python*/site-packages/tiledbvcf/libtiledbvcf.cpython-*-x86_64-linux-gnu.so | grep R*PATH | |
| echo "Check RPATH of shared object in build directory" | |
| readelf -d apis/python/build/cp311-cp311-linux_x86_64/libtiledbvcf.cpython-311-x86_64-linux-gnu.so | grep R*PATH | |
| - name: Install bcftools (for tests) | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes bcftools | |
| - name: Test tiledbvcf-py | |
| run: | | |
| python -c "import tiledbvcf; print(tiledbvcf.version)" | |
| cd apis/python | |
| pytest | |
| - name: Create a wheel | |
| run: | | |
| export LIBTILEDBVCF_PATH=$(pwd)/install-libtiledbvcf/lib/ | |
| export TileDB_DIR=$(pwd)/install-libtiledb/lib/cmake/TileDB | |
| python -m pip wheel -v --wheel-dir=apis/python/dist apis/python | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledbvcf-py-linux | |
| path: apis/python/dist/tiledbvcf-*-linux_x86_64.whl | |
| retention-days: 14 | |
| if-no-files-found: error | |
| libtiledbsoma: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb] | |
| env: | |
| TileDB_DIR: ${{ github.workspace }}/install-libtiledb | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.TILEDB_SOMA_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Setup | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes cmake | |
| - name: Configure libtiledbsoma | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/install-libtiledbsoma/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| - name: Build libtiledbsoma | |
| run: cmake --build build-libtiledbsoma -j $(nproc) | |
| - name: Install libtiledbsoma | |
| run: | | |
| cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| ls -R $(pwd)/install-libtiledb/ | |
| - name: Upload libtiledbsoma as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: | | |
| install-libtiledbsoma/* | |
| - name: Confirm linking | |
| run: ldd install-libtiledbsoma/lib/libtiledbsoma.so | |
| - name: Test libtiledbsoma | |
| run: | | |
| cmake --build build-libtiledbsoma/libtiledbsoma --target build_tests -j $(nproc) | |
| ctest --test-dir build-libtiledbsoma/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure | |
| tiledbsoma-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb, libtiledbsoma, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib" | |
| TILEDB_PATH: ${{ github.workspace }}/install-libtiledb | |
| TILEDBSOMA_PATH: ${{ github.workspace }}/install-libtiledbsoma | |
| PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig" | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.TILEDB_SOMA_REF || 'main' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbsoma | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: install-libtiledbsoma | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| # https://github.com/single-cell-data/TileDB-SOMA/blob/main/apis/python/pyproject.toml | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install \ | |
| -r apis/python/requirements_dev.txt \ | |
| "pybind11[global]>=2.10.0" \ | |
| "setuptools>=65.5.1" \ | |
| "wheel>=0.37.1" \ | |
| "cmake>=3.21" | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Install tiledbsoma-py | |
| run: | | |
| python -m pip -v install --no-build-isolation -e apis/python | |
| pip list | |
| - name: Confirm linking | |
| run: | | |
| find apis/python -name '*.so*' | |
| find apis/python -name '*.so*' | xargs ldd | |
| readelf -d apis/python/src/tiledbsoma/pytiledbsoma.*.so | grep R*PATH | |
| - name: Fail if unnecessary libtiledbsoma.so is created | |
| # https://github.com/single-cell-data/TileDB-SOMA/pull/1937 | |
| run: | | |
| if [[ -f apis/python/src/tiledbsoma/libtiledbsoma.so ]] | |
| then | |
| echo "An unnecessary libtiledbsoma.so was created during the package build" | |
| exit 1 | |
| fi | |
| - name: Test tiledbsoma-py | |
| run: | | |
| python -c "import tiledbsoma; print('successful import')" | |
| python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| python -c "import tiledbsoma; print(f'tiledbsoma-py={tiledbsoma.__version__}')" | |
| # Obtain test data | |
| cd test | |
| tar zxf soco.tgz | |
| cd .. | |
| # Run test suite | |
| PYTHONPATH=$(pwd)/apis/python/src python -m pytest apis/python/tests | |
| - name: Create a wheel | |
| run: | | |
| cd apis/python | |
| python setup.py bdist_wheel | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledbsoma-py-linux | |
| path: apis/python/dist/tiledbsoma-*-linux_x86_64.whl | |
| retention-days: 14 | |
| if-no-files-found: error | |
| tiledbsoma-r: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb, libtiledbsoma, tiledb-r] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib:${{ github.workspace }}/install-libtiledbsoma/lib" | |
| PKG_CONFIG_PATH: "${{ github.workspace }}/install-libtiledb/lib/pkgconfig:${{ github.workspace }}/install-libtiledbsoma/lib/pkgconfig" | |
| steps: | |
| - name: Clone TileDB-SOMA | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: single-cell-data/TileDB-SOMA | |
| ref: ${{ github.event.inputs.TILEDB_SOMA_REF || 'main' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-libtiledbsoma | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: install-libtiledbsoma | |
| - name: Download nightly-tiledb-r | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-r-linux | |
| - name: Bootstrap | |
| run: cd apis/r && tools/r-ci.sh bootstrap | |
| - name: Install nightly tiledb-r (and its dependencies) | |
| shell: Rscript {0} | |
| run: | | |
| install.packages("remotes") | |
| library("remotes") | |
| binary_tarball <- Sys.glob("tiledb_*_R_x86_64-pc-linux-gnu.tar.gz") | |
| message(binary_tarball) | |
| install_deps(binary_tarball, dependencies = TRUE, upgrade = "never") | |
| install.packages(binary_tarball, repos = NULL) | |
| - name: Dependencies | |
| run: cd apis/r && tools/r-ci.sh install_all | |
| - name: Build | |
| run: | | |
| export CXX17FLAGS="-Wno-deprecated-declarations -Wno-deprecated" | |
| cd apis/r && R CMD build --no-manual --no-build-vignettes . | |
| - name: Install | |
| run: | |
| cd apis/r && R CMD INSTALL --install-tests --build tiledbsoma_*.tar.gz | |
| - name: Upload R binary tarball as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledbsoma-r-linux | |
| path: apis/r/tiledbsoma_*_R_x86_64-pc-linux-gnu.tar.gz | |
| retention-days: 14 | |
| if-no-files-found: error | |
| - name: Test | |
| run: | |
| Rscript -e 'testthat::test_package("tiledbsoma")' | |
| tiledb-mariadb: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb] | |
| env: | |
| CFLAGS: -Wno-error=deprecated-declarations -Wno-error=missing-braces | |
| CXXFLAGS: -Wno-error=deprecated-declarations -Wno-error=missing-braces | |
| LD_LIBRARY_PATH: ${{ github.workspace }}/install-libtiledb/lib | |
| steps: | |
| - name: Clone MariaDB | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: MariaDB/server.git | |
| ref: mariadb-11.0.2 | |
| - name: Clone TileDB-MariaDB into MariaDB | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-MariaDB | |
| ref: ${{ github.event.inputs.TILEDB_MARIADB_REF || 'master' }} | |
| path: storage/mytile | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Configure TileDB-MariaDB | |
| run: | | |
| cmake \ | |
| -S . -B builddir \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/install-libtiledb/ \ | |
| -D PLUGIN_OQGRAPH=NO \ | |
| -D WITH_MARIABACKUP=OFF \ | |
| -D PLUGIN_TOKUDB=NO \ | |
| -D PLUGIN_ROCKSDB=NO \ | |
| -D PLUGIN_MROONGA=NO \ | |
| -D PLUGIN_SPIDER=NO \ | |
| -D PLUGIN_SPHINX=NO \ | |
| -D PLUGIN_FEDERATED=NO \ | |
| -D PLUGIN_FEDERATEDX=NO \ | |
| -D PLUGIN_CONNECT=NO \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_INCLUDE_PATH="/usr/local/include" \ | |
| -D TILEDB_FORCE_ALL_DEPS=${TILEDB_FORCE_ALL_DEPS-OFF} | |
| - name: Build TileDB-MariaDB | |
| run: | | |
| cd builddir | |
| make -j $(nproc) | |
| - name: Confirm linking | |
| run: | | |
| du -sh builddir | |
| du -sh builddir/storage/mytile | |
| ldd builddir/storage/mytile/ha_mytile.so | |
| # Fail if not properly linked to nightly libtiledb.so | |
| ldd builddir/storage/mytile/ha_mytile.so | \ | |
| grep -q $(pwd)/install-libtiledb/lib/libtiledb.so | |
| - name: Test TileDB-MariaDB | |
| run: | | |
| cd builddir | |
| ./mysql-test/mysql-test-run.pl --suite=mytile | |
| tiledb-go: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb] | |
| steps: | |
| - name: Clone TileDB-Go | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-Go | |
| ref: ${{ github.event.inputs.TILEDB_GO_REF || 'master' }} | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Install dependencies | |
| run: go get -t . | |
| - name: Test TileDB-Go | |
| run: | | |
| export CPATH=$(pwd)/install-libtiledb/include | |
| export LIBRARY_PATH=$(pwd)/install-libtiledb/lib | |
| export LD_LIBRARY_PATH=$(pwd)/install-libtiledb/lib | |
| go test -v ./... | |
| tiledb-cloud-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb, tiledb-py] | |
| env: | |
| LD_LIBRARY_PATH: "${{ github.workspace }}/install-libtiledb/lib" | |
| steps: | |
| - name: Clone TileDB-Cloud-Py | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: TileDB-Inc/TileDB-Cloud-Py | |
| ref: ${{ github.event.inputs.TILEDB_CLOUD_PY_REF || 'main' }} | |
| fetch-depth: 0 # fetch everything for python setuptools_scm | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install \ | |
| "attrs>=21.4.0" \ | |
| "certifi" \ | |
| "cloudpickle>=1.4.1,<3" \ | |
| "importlib-metadata" \ | |
| "packaging" \ | |
| "pandas>=1.2.4" \ | |
| "pyarrow>=3.0.0" \ | |
| "python-dateutil" \ | |
| "six>=1.10" \ | |
| "tblib~=1.7" \ | |
| "tiledb>=0.15.2" \ | |
| "typing-extensions" \ | |
| "urllib3>=1.26" \ | |
| "wheel" | |
| - name: Install nightly tiledb-py | |
| run: | | |
| pip install tiledb-*-linux_x86_64.whl | |
| pip list | |
| - name: Build tiledb-cloud | |
| run: | | |
| echo $LD_LIBRARY_PATH | |
| pip install --no-build-isolation . | |
| - name: Create a wheel | |
| run: python setup.py bdist_wheel | |
| - name: Upload Python wheel as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nightly-tiledb-cloud-py-linux | |
| path: dist/tiledb_cloud-*-none-any.whl | |
| retention-days: 14 | |
| if-no-files-found: error | |
| build-all: | |
| # this is simply a target to build everything locally with `act` without | |
| # creating a new release | |
| name: Build everything | |
| runs-on: ubuntu-24.04 | |
| needs: [libtiledb, tiledb-py, tiledb-r, libtiledbvcf, tiledbvcf-py, libtiledbsoma, | |
| tiledbsoma-py, tiledbsoma-r, tiledb-mariadb, tiledb-go, tiledb-cloud-py] | |
| steps: | |
| - run: echo "Success! Everything was built" | |
| create-release: | |
| name: Create a new release | |
| if: github.ref_name == 'main' && github.repository_owner == 'TileDB-Inc' && github.event_name != 'pull_request' | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| if ! gh release list | cut -f3 | grep -q $the_date | |
| then | |
| echo "Creating new release: $the_date" | |
| gh release create $the_date --notes "Nightly binaries created on $the_date" | |
| else | |
| echo "Release already exists: $the_date" | |
| fi | |
| upload-libtiledb: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, libtiledb] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-libtiledb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledb-linux | |
| path: install-libtiledb | |
| - name: Upload libtiledb to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledb -cvzf libtiledb-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledb-${the_date}.tar.gz --clobber | |
| upload-tiledb-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledb-py] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledb-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-py-linux | |
| - name: Upload tiledb-py to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| unzip -l tiledb-*-linux_x86_64.whl | |
| gh release upload ${the_date} tiledb-*-linux_x86_64.whl --clobber | |
| upload-tiledb-r: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledb-r] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledb-r | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-r-linux | |
| - name: Upload tiledb-r to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar --list -f tiledb_*_R_x86_64-pc-linux-gnu.tar.gz | |
| gh release upload ${the_date} tiledb_*_R_x86_64-pc-linux-gnu.tar.gz --clobber | |
| upload-libtiledbvcf: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, libtiledbvcf] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-libtiledbvcf | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledbvcf-linux | |
| path: install-libtiledbvcf | |
| - name: Upload libtiledbvcf to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledbvcf -cvzf libtiledbvcf-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledbvcf-${the_date}.tar.gz --clobber | |
| upload-tiledbvcf-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledbvcf-py] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledbvcf-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledbvcf-py-linux | |
| - name: Upload tiledbvcf-py to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| unzip -l tiledbvcf-*-linux_x86_64.whl | |
| gh release upload ${the_date} tiledbvcf-*-linux_x86_64.whl --clobber | |
| upload-libtiledbsoma: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, libtiledbsoma] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-libtiledbsoma | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-libtiledbsoma-linux | |
| path: install-libtiledbsoma | |
| - name: Upload libtiledbsoma to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar -C install-libtiledbsoma -cvzf libtiledbsoma-${the_date}.tar.gz ./ | |
| gh release upload ${the_date} libtiledbsoma-${the_date}.tar.gz --clobber | |
| upload-tiledbsoma-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledbsoma-py] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledbsoma-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledbsoma-py-linux | |
| - name: Upload tiledbsoma-py to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| unzip -l tiledbsoma-*-linux_x86_64.whl | |
| gh release upload ${the_date} tiledbsoma-*-linux_x86_64.whl --clobber | |
| upload-tiledbsoma-r: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledbsoma-r] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledbsoma-r | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledbsoma-r-linux | |
| - name: Upload tiledbsoma-r to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| tar --list -f tiledbsoma_*_R_x86_64-pc-linux-gnu.tar.gz | |
| gh release upload ${the_date} tiledbsoma_*_R_x86_64-pc-linux-gnu.tar.gz --clobber | |
| upload-tiledb-cloud-py: | |
| runs-on: ubuntu-24.04 | |
| needs: [create-release, tiledb-cloud-py] | |
| permissions: | |
| contents: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TZ: "America/New_York" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nightly-tiledb-cloud-py | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nightly-tiledb-cloud-py-linux | |
| - name: Upload tiledb-cloud-py to release | |
| run: | | |
| the_date=$(date +%Y-%m-%d) | |
| unzip -l tiledb_cloud-*-none-any.whl | |
| gh release upload ${the_date} tiledb_cloud-*-none-any.whl --clobber | |
| issue: | |
| permissions: | |
| issues: write | |
| runs-on: ubuntu-24.04 | |
| needs: [build-all] | |
| if: ( failure() || cancelled() ) && github.repository_owner == 'TileDB-Inc' && github.event_name == 'schedule' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Open Issue | |
| uses: TileDB-Inc/github-actions/open-issue@main | |
| with: | |
| name: centralized nightlies | |
| label: nightly-failure | |
| assignee: jdblischak | |
| env: | |
| TZ: "America/New_York" |