python.yml

name: Python build

on:
  push:
    branches:
      - master
    paths:
      - 'common/**'
      - 'spark/**'
      - 'spark-shaded/**'
      - 'pom.xml'
      - 'python/**'
      - '.github/workflows/python.yml'
  pull_request:
    branches:
      - '*'
    paths:
      - 'common/**'
      - 'spark/**'
      - 'spark-shaded/**'
      - 'pom.xml'
      - 'python/**'
      - '.github/workflows/python.yml'

env:
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=60
  JAI_CORE_VERSION: "1.1.3"
  JAI_CODEC_VERSION: "1.1.3"
  JAI_IMAGEIO_VERSION: "1.1"
  DO_NOT_TRACK: true

permissions:
  contents: read

jobs:
  build:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        include:
          - spark: '3.5.0'
            scala: '2.12.8'
            python: '3.10'
            hadoop: '3'
            shapely: '1'
          - spark: '3.5.0'
            scala: '2.12.8'
            python: '3.10'
            hadoop: '3'
          - spark: '3.5.0'
            scala: '2.12.8'
            python: '3.9'
            hadoop: '3'
          - spark: '3.5.0'
            scala: '2.12.8'
            python: '3.8'
            hadoop: '3'
          - spark: '3.4.0'
            scala: '2.12.8'
            python: '3.10'
            hadoop: '3'
          - spark: '3.4.0'
            scala: '2.12.8'
            python: '3.9'
            hadoop: '3'
          - spark: '3.4.0'
            scala: '2.12.8'
            python: '3.8'
            hadoop: '3'
          - spark: '3.4.0'
            scala: '2.12.8'
            python: '3.7'
            hadoop: '3'
          - spark: '3.4.0'
            scala: '2.12.8'
            python: '3.7'
            hadoop: '3'
            shapely: '1'
          - spark: '3.3.0'
            scala: '2.12.8'
            python: '3.8'
            hadoop: '3'
          - spark: '3.2.0'
            scala: '2.12.8'
            python: '3.7'
            hadoop: '2.7'
          - spark: '3.1.2'
            scala: '2.12.8'
            python: '3.7'
            hadoop: '2.7'
          - spark: '3.0.3'
            scala: '2.12.8'
            python: '3.7'
            hadoop: '2.7'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'zulu'
          java-version: '8'
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: Cache Maven packages
        uses: actions/cache@v3
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2
      - env:
          SPARK_VERSION: ${{ matrix.spark }}
          SCALA_VERSION: ${{ matrix.scala }}
        run: |
            SPARK_COMPAT_VERSION="3.0"
            if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
              SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
            fi
            mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
      - env:
          SPARK_VERSION: ${{ matrix.spark }}
          HADOOP_VERSION: ${{ matrix.hadoop }}
        run: |
          wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
          wget https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
          wget https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
          wget https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
          tar -xzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
          mv -v jai_core-${JAI_CORE_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
          mv -v jai_codec-${JAI_CODEC_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
      - run: sudo apt-get -y install python3-pip python-dev-is-python3
      - run: sudo pip3 install -U setuptools
      - run: sudo pip3 install -U wheel
      - run: sudo pip3 install -U virtualenvwrapper
      - run: python3 -m pip install pipenv
      - run: cd python; python3 setup.py build_ext --inplace
      - env:
          SPARK_VERSION: ${{ matrix.spark }}
          PYTHON_VERSION: ${{ matrix.python }}
          SHAPELY_VERSION: ${{ matrix.shapely }}
        run: |
          cd python
          if [ "${SHAPELY_VERSION}" == "1" ]; then
            echo "Patching Pipfile to use Shapely 1.x"
            sed -i 's/^shapely.*$/shapely="<2.0.0"/g' Pipfile
          fi
          pipenv --python ${PYTHON_VERSION}
          pipenv install pyspark==${SPARK_VERSION}
          pipenv install --dev
          pipenv graph
      - env:
          SPARK_VERSION: ${{ matrix.spark }}
          HADOOP_VERSION: ${{ matrix.hadoop }}
        run: find spark-shaded/target -name sedona-*.jar -exec cp {} spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/ \;
      - env:
          SPARK_VERSION: ${{ matrix.spark }}
          HADOOP_VERSION: ${{ matrix.hadoop }}
        run: (export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION};export PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)