ci-build.yml

name: CI
on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main
jobs:
  tests:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
    name: Python ${{ matrix.python-version }} Build & Tests
    steps:
      - name: Add apt repo
        run: sudo add-apt-repository universe

      # Setup Java & Python
      - name: Setup Java
        uses: actions/setup-java@v1
        with:
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64

      - name: Checkout DataGateway API
        uses: actions/checkout@v2

      # Install Nox, Poetry and API's dependencies
      - name: Install Nox
        run: pip install nox==2020.8.22
      - name: Install Poetry
        run: pip install poetry==1.1.9

      # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
      - name: Uninstall setuptools
        run: poetry run pip uninstall -y setuptools
      - name: Install older setuptools
        run: poetry run pip install 'setuptools<58.0.0'

      # This command is a workaround for getting Poetry working with Python 3.10. An
      # fix is made in Poetry 1.2.0a2 but there is currently no official release for
      # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the
      # experimental installer is a workaround for Poetry 1.1.x
      # See https://github.com/python-poetry/poetry/issues/4210 for more details
      - name: Disable Poetry's experimental new installer
        run: poetry config experimental.new-installer false

      - name: Install dependencies
        run: poetry install

      # Prep for using the API for tests
      - name: Create log file
        run: touch logs.log
      - name: Configure log file location
        run: echo "`yq \
          '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example
      - name: Configure datagateway extension
        run: echo "`yq \
          '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
      - name: Create config.yaml
        run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
      - name: Create search_api_mapping.json
        run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json

      # Run Unit tests
      - name: Run Nox unit tests session
        run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml
      - name: Upload unit test code coverage report
        if: matrix.python-version == '3.6'
        uses: codecov/codecov-action@v3

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        if: success()
        uses: actions/checkout@v2
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests
      - name: Add anon user to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install

      - name: Add dummy data to icatdb
        run: |
          poetry run python -m util.icat_db_generator

      # Run Nox integration tests session, saves and uploads a coverage report to codecov
      - name: Run Nox Integration tests session
        if: success()
        run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml
      - name: Upload integration test code coverage report
        if: matrix.python-version == '3.6'
        uses: codecov/codecov-action@v3

  linting:
    runs-on: ubuntu-20.04
    name: Linting
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9.7"
          architecture: x64
      - name: Checkout DataGateway API
        uses: actions/checkout@v2

      - name: Install Nox
        run: pip install nox==2020.8.22
      - name: Install Poetry
        run: pip install poetry==1.1.9

      - name: Run Nox lint session
        run: nox -s lint

  formatting:
    runs-on: ubuntu-20.04
    name: Code Formatting
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9.7"
          architecture: x64
      - name: Checkout DataGateway API
        uses: actions/checkout@v2

      - name: Install Nox
        run: pip install nox==2020.8.22
      - name: Install Poetry
        run: pip install poetry==1.1.9

      - name: Run Nox black session
        run: nox -s black

  safety:
    runs-on: ubuntu-20.04
    name: Dependency Safety
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9.7"
          architecture: x64
      - name: Checkout DataGateway API
        uses: actions/checkout@v2

      - name: Install Nox
        run: pip install nox==2020.8.22
      - name: Install Poetry
        run: pip install poetry==1.1.9

      - name: Run Nox safety session
        run: nox -s safety

  generator-script-testing:
    runs-on: ubuntu-20.04
    continue-on-error: true
    name: icatdb Generator Script Consistency Test
    steps:
      - name: Add apt repo
        run: sudo add-apt-repository universe

      # Setup Java & Python
      - name: Setup Java
        uses: actions/setup-java@v1
        with:
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9.7"
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@v2
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      - name: Checkout DataGateway API
        uses: actions/checkout@v2

      - name: Create config.yaml
        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
      - name: Create search_api_mapping.json
        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
      - name: Install Poetry
        run: pip install poetry==1.1.9

      # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
      - name: Uninstall setuptools
        run: poetry run pip uninstall -y setuptools
      - name: Install older setuptools
        run: poetry run pip install 'setuptools<58.0.0'

      - name: Install dependencies
        run: poetry install

      - name: Add dummy data to icatdb
        run: poetry run python -m util.icat_db_generator
      - name: Get SQL dump of dummy data
        run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_1.sql

      # Drop and re-create icatdb to remove generated data
      - name: Drop icatdb
        run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;'
      - name: Create icatdb
        run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;'
      # Regenerate table structure of icatdb
      - name: Reinstall ICAT Server
        run: cd /home/runner/install/icat.server; ./setup -vv install

      - name: Add (new) dummy data to icatdb
        run: |
          cd /home/runner/work/datagateway-api/datagateway-api; poetry run python -m util.icat_db_generator
      - name: Get SQL dump of new dummy data
        run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_2.sql

      # Tests that the generator script produces consistent data over two separate runs
      - name: Diff SQL dumps
        run: diff -s ~/generator_script_dump_1.sql ~/generator_script_dump_2.sql

      # Drop and re-create icatdb to remove generated data
      - name: Drop icatdb
        run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;'
      - name: Create icatdb
        run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;'
      # Regenerate table structure of icatdb
      - name: Reinstall ICAT Server
        run: cd /home/runner/install/icat.server; ./setup -vv install

      - name: Checkout DataGateway API (default branch)
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.repository.default_branch }}

      - name: Create config.yaml
        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
      - name: Create search_api_mapping.json
        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json

      # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
      - name: Uninstall setuptools
        run: poetry run pip uninstall -y setuptools
      - name: Install older setuptools
        run: poetry run pip install 'setuptools<58.0.0'

      - name: Install dependencies
        run: poetry install

      - name: Add dummy data to icatdb
        run: poetry run python -m util.icat_db_generator
      - name: Get SQL dump of dummy data from main's generator script
        run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_main.sql

      # Tests that the generator script produces the same data as is produced with main's version

      # NOTE: If a delibrate change is made to the script that will change the data generated,
      # the diff (and therefore this job) will fail. If this is the case, don't be alarmed.
      # The `continue-on-error` keyword has been added to this job so the workflow should
      # pass when the PR is merged in, even if this job fails.
      # But, if you didn't mean to change the output of the script, there is likely a
      # problem with the changes made that will make the generator script's data
      # different to SciGateway preprod
      - name: Diff SQL dumps
        run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql