hi-ml-pr.yml

name: HI-ML HI-ML-Azure PR
on:
  push:
    branches: [ main ]
    tags:
      - '*'
  pull_request:
    branches: [ main ]
    paths:
      - "hi-ml-azure/**"
      - "hi-ml/**"
      - ".github/workflows/hi-ml-pr.yml"

# Cancel previous runs of this workflow that are still in progress.
concurrency:
  group: ${{ github.ref }}/hi-ml-pr
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

permissions:
  # This is required for requesting the Azure login token
  id-token: write
  # This is required for actions/checkout
  contents: read

env:
  pythonVersion: 3.9
  HIML_TENANT_ID: ${{ secrets.HIML_TENANT_ID }}
  HIML_RESOURCE_GROUP: ${{ secrets.HIML_RESOURCE_GROUP }}
  HIML_SUBSCRIPTION_ID: ${{ secrets.HIML_SUBSCRIPTION_ID }}
  HIML_WORKSPACE_NAME: ${{ secrets.HIML_WORKSPACE_NAME }}
  HIML_SERVICE_PRINCIPAL_ID: ${{ secrets.HIML_SERVICE_PRINCIPAL_ID }}
  HIML_DIST_ARTIFACT_SUFFIX: '-dist'
  HIML_PACKAGE_NAME_ARTIFACT_SUFFIX: '-package_name'
  HIML_VERSION_ARTIFACT_SUFFIX: '-latest_version'
  # Set the AML experiment name for all AML jobs submitted during tests. Github.ref looks like
  # "refs/pull/123/merge" for PR builds.
  HIML_EXPERIMENT_NAME: ${{ github.ref }}
  PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
  PYPI_TEST_API_TOKEN: ${{ secrets.PYPI_TEST_API_TOKEN }}

jobs:

  cancel-azureml:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Cancel previous AzureML runs
        uses: ./.github/actions/cancel_azureml_jobs

  himl-flake8:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: flake8
        run: |
          make pip_test
          make flake8

  himl-mypy:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: mypy
        run: |
          make pip_test
          make mypy

  # himl-pyright:
  #   runs-on: ubuntu-20.04
  #   steps:
  #     - uses: actions/checkout@v3
  #       with:
  #         lfs: true
  #     - uses: actions/setup-node@v3
  #       with:
  #         node-version: '14'
  #     - uses: conda-incubator/setup-miniconda@v2
  #     - name: pyright
  #       shell: bash -l {0}
  #       run: |
  #         conda info
  #         make pyright

  himl-pytest-fast:
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
      # This will let all parts of the matrix run, to collect as many errors as possible, rather than aborting after first fail
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Install hi-ml-azure in editable mode
        if: ${{ matrix.folder != 'hi-ml-azure' }}
        run: |
          cd hi-ml-azure

          # Install local package in editable mode
          make pip_local

      - name: Install hi-ml in editable mode
        if: ${{ matrix.folder != 'hi-ml-azure' && matrix.folder != 'hi-ml' }}
        run: |
          cd hi-ml

          # Install local package in editable mode
          make pip_local

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Test with pytest, fast only
        run: |
          cd ${{ matrix.folder }}

          make pip_test
          # Install local package in editable mode
          make pip_local

          # Run fast tests
          make pytest_fast

  himl-build-python:
    runs-on: ubuntu-20.04
    needs: [ himl-pytest-fast ]
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Build Distribution
        run: |
          cd ${{ matrix.folder }}
          make build

      - name: Upload package artifacts
        uses: ./.github/actions/upload_package_artifacts
        with:
          folder: ${{ matrix.folder }}

  himl-test-artifact-pkg:
    runs-on: ubuntu-20.04
    needs: [ himl-build-python, cancel-azureml ]
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Download dependent package artifacts
        id: download_dep
        if: ${{ matrix.folder != 'hi-ml-azure' }}
        uses: ./.github/actions/download_package_artifacts
        with:
          folder: hi-ml-azure

      - name: Install dependencies
        if: ${{ matrix.folder != 'hi-ml-azure' }}
        run: |
          cd hi-ml-azure

          make pip_upgrade

          # Install package
          pip install ${{ steps.download_dep.outputs.package_filename }}

      - name: Download package artifacts
        id: download
        uses: ./.github/actions/download_package_artifacts
        with:
          folder: ${{ matrix.folder }}

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Install artifact and test
        run: |
          cd ${{ matrix.folder }}

          make pip_upgrade

          # Install package
          pip install ${{ steps.download.outputs.package_filename }}

          # Set env vars so wheel is sent to azureml as a private package
          # This had to be deprecated because the upload always uses SAS tokens encrypted with the now disabled storage account key
          # echo "HIML_AZURE_WHEEL_FILENAME=${{ steps.download.outputs.package_filename }}" >> $GITHUB_ENV

          # Test with pytest
          make pytest_and_coverage

      - name: Upload coverage report to Codecov
        # Coverage should also be uploaded if tests still fail
        if: always()
        uses: codecov/codecov-action@v3
        with:
            flags: ${{ matrix.folder }}

  himl_smoke_helloworld_v1:
    runs-on: ubuntu-20.04
    needs: [ cancel-azureml ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: ./.github/actions/prepare_himl_python_env
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Run smoke_helloworld_v1
        run: |
          cd hi-ml
          make smoke_helloworld_v1

  himl_smoke_helloworld_v2:
    runs-on: ubuntu-20.04
    needs: [ cancel-azureml ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: ./.github/actions/prepare_himl_python_env
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Run smoke_helloworld_v2
        run: |
          cd hi-ml
          make smoke_helloworld_v2

  himl_smoke_helloworld_v1_2nodes:
    runs-on: ubuntu-20.04
    needs: [ cancel-azureml ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: ./.github/actions/prepare_himl_python_env
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Run smoke_helloworld_v1_2nodes
        run: |
          cd hi-ml
          make smoke_helloworld_v1_2nodes

  himl_smoke_helloworld_v2_2nodes:
    runs-on: ubuntu-20.04
    needs: [ cancel-azureml ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: ./.github/actions/prepare_himl_python_env
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Run smoke_helloworld_v2_2nodes
        run: |
          cd hi-ml
          make smoke_helloworld_v2_2nodes

  himl-smoke-tests-completed:
    # This job is just a placeholder to ensure that all smoke tests have completed before
    # publishing the package. Reference this job rather than the individual smoke tests.
    runs-on: ubuntu-20.04
    needs: [
         himl_smoke_helloworld_v1,
         himl_smoke_helloworld_v2,
         himl_smoke_helloworld_v1_2nodes,
         himl_smoke_helloworld_v2_2nodes,
      ]
    steps:
      - name: Smoke tests completed
        run: |
          echo "All hi-ml smoke tests have passed"

  himl-publish-testpypi-pkg:
    runs-on: ubuntu-20.04
    needs: [ himl-test-artifact-pkg, himl-mypy, himl-flake8, himl-smoke-tests-completed ]
    if: "!startsWith(github.ref, 'refs/tags/v')"
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Download package artifacts
        id: download
        uses: ./.github/actions/download_package_artifacts
        with:
          folder: ${{ matrix.folder }}

      - name: Publish package to TestPyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: __token__
          password: ${{ env.PYPI_TEST_API_TOKEN }}
          packages_dir: ${{ matrix.folder }}/dist/
          repository_url: https://test.pypi.org/legacy/
          verbose: true

      - name: Show published TestPyPI package version
        run: |
          echo "Install this package version via:"
          echo "pip install --extra-""index-url https://test.pypi.org/simple ${{ steps.download.outputs.package_version }}"

  himl-publish-pypi-pkg:
    runs-on: ubuntu-20.04
    needs: [ himl-test-artifact-pkg, himl-mypy, himl-flake8, himl-smoke-tests-completed ]
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Download package artifacts
        id: download
        uses: ./.github/actions/download_package_artifacts
        with:
          folder: ${{ matrix.folder }}

      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: __token__
          password: ${{ env.PYPI_API_TOKEN }}
          packages_dir: ${{ matrix.folder }}/dist/

  himl-test-pypi-pkg:
    runs-on: ubuntu-20.04
    needs: [ himl-publish-pypi-pkg ]
    strategy:
      matrix:
        folder: [ hi-ml, hi-ml-azure ]
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: true

      - name: Set up Python ${{ env.pythonVersion }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}

      - name: Download package artifacts
        id: download
        uses: ./.github/actions/download_package_artifacts
        with:
          folder: ${{ matrix.folder }}

      - name: Install dependencies
        if: ${{ matrix.folder != 'hi-ml-azure' }}
        run: |
          cd hi-ml-azure

          # Install local package in editable mode
          make pip_local

      - name: Azure login using OpenID Connect
        uses: ./.github/actions/azure-login

      - name: Install PyPI package and test
        run: |
          cd ${{ matrix.folder }}

          make pip_upgrade
          make pip_test

          while ! pip install ${{ steps.download.outputs.package_version }}
          do
            echo waiting for pypi
            sleep 10
          done

          # Test with pytest, fast only
          make pytest_fast

  hi-ml-pr-completed:
    runs-on: ubuntu-20.04
    if: "!startsWith(github.ref, 'refs/tags/v')"
    needs: [ himl-publish-testpypi-pkg ]
    steps:
      - name: Final PR Job
        run: |
          echo "All hi-ml PR jobs have passed"