ci.yml

# JULEA - Flexible storage framework
# Copyright (C) 2019-2021 Michael Kuhn
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

name: CI
on: [push, pull_request]
jobs:
  style:
    name: Style
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Check format
        run: |
          ./scripts/format.sh
      - name: Check shell scripts
        run: |
          ./scripts/check.sh
  dependencies:
    name: Dependencies
    runs-on: ${{ matrix.os.dist }}
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        os:
          - dist: ubuntu-22.04
            compiler: gcc
            compiler_version: 11.2.0
          - dist: ubuntu-22.04
            compiler: clang
            compiler_version: 14.0.0
          - dist: ubuntu-20.04
            compiler: gcc
            compiler_version: 9.4.0
          - dist: ubuntu-20.04
            compiler: clang
            compiler_version: 11.0.0
          - dist: ubuntu-18.04
            compiler: gcc
            compiler_version: 7.5.0
          - dist: ubuntu-18.04
            compiler: clang
            compiler_version: 9.0.0
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v3
        with:
          path: dependencies
          key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Install dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        env:
          JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }}
        run: |
          ./scripts/install-dependencies.sh
  build:
    name: Build
    needs: dependencies
    runs-on: ${{ matrix.os.dist }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - dist: ubuntu-22.04
            compiler: gcc
            compiler_version: 11.2.0
          - dist: ubuntu-22.04
            compiler: clang
            compiler_version: 14.0.0
          - dist: ubuntu-20.04
            compiler: gcc
            compiler_version: 9.4.0
          - dist: ubuntu-20.04
            compiler: clang
            compiler_version: 11.0.0
          - dist: ubuntu-18.04
            compiler: gcc
            compiler_version: 7.5.0
          - dist: ubuntu-18.04
            compiler: clang
            compiler_version: 9.0.0
        dependencies: [system, spack]
        exclude:
          # GLib is too old for TAP on Ubuntu 18.04
          - os:
              dist: ubuntu-18.04
            dependencies: system
          # libfabric is too old on Ubuntu 20.04
          - os:
              dist: ubuntu-20.04
            dependencies: system
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Clean up system packages
        # We need to remove glib-network, otherwise libproxy might cause crashes.
        run: |
          sudo apt --yes purge glib-networking
          sudo apt --yes --purge autoremove
          sudo aa-remove-unknown
      - name: Install dependencies
        if: ${{ matrix.dependencies == 'system' }}
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
          if test "${{ matrix.os.dist }}" = 'ubuntu-22.04'
          then
            sudo apt --yes --no-install-recommends install meson
          else
            sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel
            sudo pip3 install meson
          fi
      - name: Cache dependencies
        id: cache
        if: ${{ matrix.dependencies == 'spack' }}
        uses: actions/cache@v3
        with:
          path: dependencies
          key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure
        env:
          CC: ${{ matrix.os.compiler }}
        run: |
          . scripts/environment.sh
          meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release --werror bld
      - name: Build
        run: |
          . scripts/environment.sh
          ninja -C bld
      - name: Install
        run: |
          . scripts/environment.sh
          ninja -C bld install
  tests:
    name: Tests
    needs: dependencies
    runs-on: ${{ matrix.os.dist }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - dist: ubuntu-20.04
            compiler: gcc
            compiler_version: 9.4.0
          - dist: ubuntu-20.04
            compiler: clang
            compiler_version: 11.0.0
          - dist: ubuntu-18.04
            compiler: gcc
            compiler_version: 7.5.0
          - dist: ubuntu-18.04
            compiler: clang
            compiler_version: 9.0.0
        dependencies: [system, spack]
        julea:
          # Default
          - object: posix
            kv: lmdb
            db: sqlite
          # Object backends
          - object: gio
            kv: lmdb
            db: sqlite
          # KV backends
          - object: posix
            kv: mongodb
            db: sqlite
          - object: posix
            kv: leveldb
            db: sqlite
          - object: posix
            kv: rocksdb
            db: sqlite
          - object: posix
            kv: sqlite
            db: sqlite
          # DB backends
          - object: posix
            kv: lmdb
            db: memory
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mysql
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mariadb
        exclude:
          # GLib is too old for TAP on Ubuntu 18.04
          - os:
              dist: ubuntu-18.04
            dependencies: system
          # libfabric is too old on Ubuntu 20.04
          - os:
              dist: ubuntu-20.04
            dependencies: system
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Clean up system packages
        # We need to remove glib-network, otherwise libproxy might cause crashes.
        run: |
          sudo apt --yes purge glib-networking
          sudo apt --yes --purge autoremove
          sudo aa-remove-unknown
      - name: Install dependencies
        if: ${{ matrix.dependencies == 'system' }}
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
          sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel
          sudo pip3 install meson
      - name: Cache dependencies
        id: cache
        if: ${{ matrix.dependencies == 'spack' }}
        uses: actions/cache@v3
        with:
          path: dependencies
          key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Set up MongoDB
        if: ${{ matrix.julea.kv == 'mongodb' }}
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install mongodb-server
          sudo systemctl start mongodb.service
      - name: Set up MySQL
        if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' }}
        run: |
          sudo systemctl start mysql.service
          mysql --user=root --password=root --execute="CREATE DATABASE juleadb;"
          mysql --user=root --password=root --execute="CREATE USER 'julea'@'localhost' IDENTIFIED WITH mysql_native_password BY 'aeluj';"
          mysql --user=root --password=root --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"
      - name: Set up MariaDB
        if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' }}
        # FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
        # Ubuntu 18.04 has MySQL 5.7, Ubuntu 20.04 has MySQL 8.0
        run: |
          if test "${{ matrix.os.dist }}" = 'ubuntu-18.04'
          then
            sudo apt --yes purge mysql-client mysql-client-5.7 mysql-server mysql-server-5.7
          else
            sudo apt --yes purge mysql-client mysql-client-8.0 mysql-server mysql-server-8.0
          fi
          sudo apt --yes --purge autoremove
          sudo aa-remove-unknown
          sudo rm --recursive --force /var/lib/mysql
          sudo apt update || true
          sudo apt --yes --no-install-recommends install mariadb-server
          sudo systemctl start mariadb.service
          sudo mariadb --execute="CREATE DATABASE juleadb;"
          sudo mariadb --execute="CREATE USER 'julea'@'localhost' IDENTIFIED BY 'aeluj';"
          sudo mariadb --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"
      - name: Configure
        env:
          CC: ${{ matrix.os.compiler }}
        # FIXME Ubuntu 18.04's sanitizers produce many false positives.
        run: |
          . scripts/environment.sh
          if test "${{ matrix.os.dist }}" = 'ubuntu-18.04'
          then
            meson setup bld
          elif test "${{ matrix.os.compiler }}" = 'clang'
          then
            meson setup -Db_sanitize=address,undefined -Db_lundef=false bld
          else
            meson setup -Db_sanitize=address,undefined bld
          fi
      - name: Build
        run: |
          . scripts/environment.sh
          ninja -C bld
      - name: Create configuration
        # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist.
        run: |
          . scripts/environment.sh
          JULEA_KV_COMPONENT='server'
          if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_COMPONENT='client'; fi
          JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}"
          if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi
          JULEA_DB_COMPONENT='server'
          if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_COMPONENT='client'; fi
          JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}"
          if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}"
      - name: Tests
        env:
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          sleep 10
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: HDF5 Tests
        if: ${{ matrix.julea.db != 'memory' }}
        env:
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          export HDF5_VOL_CONNECTOR=julea-kv
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          sleep 10
          export HDF5_VOL_CONNECTOR=julea-db
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          ./scripts/setup.sh stop
  doxygen:
    name: Doxygen
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install dependencies
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install doxygen
      - name: Generate documentation
        run: |
          doxygen
      - name: Upload doxygen
        uses: actions/upload-artifact@v3
        with:
          name: doxygen
          path: html
  coverage:
    # FIXME Reduce redundancies
    # FIXME Increase coverage (currently, only one configuration runs)
    name: Coverage
    needs: dependencies
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v3
        with:
          path: dependencies
          key: ubuntu-20.04-gcc-9.4.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure
        env:
          CC: gcc
        run: |
          . scripts/environment.sh
          meson setup -Db_coverage=true -Db_sanitize=address,undefined bld
      - name: Build
        run: |
          . scripts/environment.sh
          ninja -C bld
      - name: Create configuration
        run: |
          . scripts/environment.sh
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-component=server --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-component=server --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-component=server --db-path=/tmp/julea/db/sqlite
      - name: Tests
        env:
          HDF5_VOL_CONNECTOR: julea-db
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: Generate coverage
        run: |
          . scripts/environment.sh
          ninja -C bld coverage
      - name: Upload coverage
        uses: actions/upload-artifact@v3
        with:
          name: coverage
          path: bld/meson-logs/coveragereport
  publish:
    name: Publish
    needs: [doxygen, coverage]
    runs-on: ubuntu-22.04
    permissions:
      contents: write
    if: ${{ github.ref == 'refs/heads/master' }}
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: gh-pages
          path: pages
      - name: Download doxygen
        uses: actions/download-artifact@v3
        with:
          name: doxygen
          path: doxygen
      - name: Download coverage
        uses: actions/download-artifact@v3
        with:
          name: coverage
          path: coverage
      - name: Prepare
        run: |
          rsync --archive --human-readable --verbose --delete --exclude=.git doxygen/. pages
          mkdir pages/coverage
          rsync --archive --human-readable --verbose --delete --exclude=.git coverage/. pages/coverage
          touch pages/.nojekyll
      - name: Commit
        run: |
          cd pages
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          git add --all .
          git commit --amend --reset-author --message='Pages for ${{ github.sha }}'
          git push --force