.gitlab-ci.yml

include:
    - project: "epi2melabs/ci-templates"
      file: "push-conda.yaml"
    - project: "epi2melabs/ci-templates"
      file: "snippets.yaml"


image: ${UBUNTU_IMAGE}:20.04

stages:
    - test
    - build
    - release

# Install a particular Python and set PYTHON envvar for Makefile 
.install-pyenv: &install-pyenv |
    git clone https://github.com/pyenv/pyenv.git ~/.pyenv
    export PYENV_ROOT="$HOME/.pyenv"
    export PATH="$PYENV_ROOT/bin:$PATH"
    PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install ${PYENV_VERSION}
    export PYTHON=$(PYENV_VERSION=$PYENV_VERSION pyenv which python)

.preparetest: &preparetest
    stage: test
    before_script:
        - export DEBIAN_FRONTEND=noninteractive
        - apt-get update -qq
        - apt-get install -y -qq 
          gcc make cmake curl wget git
          zlib1g-dev libbz2-dev liblzma-dev libncurses5-dev libcurl4-gnutls-dev
          libssl-dev libffi-dev valgrind
          libreadline8 libreadline-dev sqlite3 libsqlite3-dev
        - *install-pyenv


.minimal-python: &minimal-python
    before_script:
        - export DEBIAN_FRONTEND=noninteractive
        - apt-get update -qq && apt-get install -y -qq
          python3-all-dev python3-venv


# Check that the changelog is up to date and other stuff
# do this is test 'cos no one wants to wait for all the
# wheels to build to find out they forgot to update the log
deploy-checks:
    stage: test
    script:
        - export LICENSE_FILE="ONT-LICENSE"
        - !reference [.check, python-version]
        - !reference [.check, changelog]
        - !reference [.check, license]
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/'



# medaka test suite on various Pythons, latest as of 2024-08-14
pytest: &pytest
    <<: *preparetest
    variables:
        MEDAKA_CPU: 1
    script:
        - make test
        - make test-consensus
        - make test-variants
    parallel:
        matrix:
            - PYENV_VERSION: 3.8.19
            - PYENV_VERSION: 3.9.19
            - PYENV_VERSION: 3.10.14
            - PYENV_VERSION: 3.11.9
            # no edlib for 3.12, and bdist_wheel fails
            #- PYENV_VERSION: 3.12.5


test-mem-check:
    variables:
        PYENV_VERSION: 3.8.18
    <<: *preparetest
    script:
        - make mem_check


###
# Source distribution
#

build:sdist:
    stage: build
    <<: *minimal-python
    script:
        - make sdist
    artifacts:
        paths:
            - dist/*.tar.gz

###
# Many linux builds
#
.many-linux:
    stage: build
    variables:
        DO_COUNT_TEST: 1
        MEDAKA_CPU: 1
    script:
        - echo "Building a Python ${PYWHEEL} wheel on manylinux_${FLAVOUR} for ${COMPUTE}"
        - if [[ "${COMPUTE}" == "cpu" ]]; then export PACKAGE_NAME=medaka-cpu; sed -i "s/torch==[0-9]\+[\.0-9]\+/&+cpu/" requirements.txt; fi
        - cat requirements.txt
        - ./build-wheels.sh . ${PYWHEEL}
    artifacts:
        paths:
            - wheelhouse-final/*.whl


wheels-2014:
    extends: .many-linux
    image: "quay.io/pypa/manylinux2014_x86_64"
    parallel:
        matrix:
            - PYWHEEL: [8, 9]
              COMPUTE: ["gpu", "cpu"]
              FLAVOUR: ["2014"]


wheels-2_28:
    extends: .many-linux
    image: "quay.io/pypa/manylinux_2_28_x86_64"
    before_script: 
        - dnf install -y libffi-devel
    parallel:
        matrix:
            - PYWHEEL: [8, 9, 10, 11]
              COMPUTE: ["gpu", "cpu"]
              FLAVOUR: ["2_28"]


wheels-arm-2014:
    extends: .many-linux
    tags:
        - arm64
    image: quay.io/pypa/manylinux2014_aarch64
    before_script:
        # ARM CI image doesn't have git lfs, so need to jump some hoops
        - yum install -y wget
        - ROOT_DIR=$(pwd)
        - mkdir git-lfs && cd git-lfs
          && wget https://github.com/git-lfs/git-lfs/releases/download/v3.5.1/git-lfs-linux-arm64-v3.5.1.tar.gz
          && tar -xzvf git-lfs-linux-arm64-v3.5.1.tar.gz
          && cd git-lfs-3.5.1 && chmod +x install.sh && ./install.sh
        - cd ${ROOT_DIR}
        - GIT_SSL_NO_VERIFY=1 git lfs pull
        # pytorch 2.3 wheel is busted with something looking like pytorch/issues/97226:
        #   we get Illegal instruction when running medaka counts test, so use 2.0.1 which
        #   is known to work
        - sed -i '/^torch~=[^[:space:]]*/s/^torch~=[^[:space:]]*/torch==2.0.1/' requirements.txt
    parallel:
        matrix:
            - PYWHEEL: [8, 9, 10]
              COMPUTE: ["gpu"]
              FLAVOUR: ["2014"]


###
# macOS wheels, note we don't take special care of macOS runtime version
#

.mac-wheel:
    stage: build
    script:
        - echo "Building a wheel on macOS"
        - arch; echo ""
        - clang --version
        - gcc --version
        - python3 -m venv pypi_build --prompt "pypi"
        - . ./pypi_build/bin/activate
        - pip install cibuildwheel
        - make scripts/mini_align
        - CIBW_BUILD=cp3${PYWHEEL}-macosx_${arch} MACOSX_DEPLOYMENT_TARGET=12 CIBW_ENVIRONMENT="MEDAKA_LINK_ARGS='-L/opt/homebrew/lib' MEDAKA_DIST=1" cibuildwheel --output-dir wheelhouse-final
    artifacts:
        paths:
            - wheelhouse-final/*.whl
    parallel:
        matrix:
            # 8 gives grief with wanted to add -mtune=haswell \:D/
            # suggests platform.machine() isn't reported correctly
            - PYWHEEL: [9, 10, 11]


macOS-arm64-wheel:
    extends: .mac-wheel
    variables:
        arch: arm64
    tags:
        - macos
        - oxford
        - arm
        - shell


# macOS-x64-wheel:
# there's no pytorch macOS x64 wheels, so we don't build them


# Build conda packages but don't publish them
.before-conda: &before-conda |
    export CONDA_PKG=${CI_PROJECT_NAME}
    export CONDA_PKG_VERSION="dev"
    cd conda

conda:
    extends: .deploy-conda
    stage: build
    variables:
        UPLOAD: "no"
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda
    rules:
        - if: $CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/
          when: never
        - if: $CI_PIPELINE_SOURCE == "push"

conda-arm:
    extends: .deploy-conda-linux-arm
    stage: build
    variables:
        UPLOAD: "no"
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda
    rules:
        - if: $CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/
          when: never
        - if: $CI_PIPELINE_SOURCE == "push"

conda-mac-arm:
    extends: .deploy-conda-mac-arm
    stage: build
    variables:
        UPLOAD: "no"
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda
    rules:
        - if: $CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/
          when: never
        - if: $CI_PIPELINE_SOURCE == "push"


###
# Deploy stages
#

# Send all tags to PyPI
push-pypi:
    stage: release
    <<: *minimal-python
    script:
        - make pypi_build/bin/activate
        - source pypi_build/bin/activate
        - twine upload dist/*.tar.gz wheelhouse-final/*.whl
    only:
        - tags
    artifacts:
        # just gathered together for easy inspection
        paths:
            - dist/*.tar.gz
            - wheelhouse-final/*.whl


.setup-gh-push:
    before_script:
        - apt-get update -qq && apt-get install -y -qq
          git python3-all-dev python3-venv git-lfs
        - mkdir ~/.ssh/
        - cp $RESEARCH_BOT_GH_KEY ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
        - echo -e "Host github.com\n\tStrictHostKeyChecking no\n\tHostname ssh.github.com\n\tPort 443\n\n" > ~/.ssh/config
        - git config --global user.email "research.git@nanoporetech.com"
        - git config --global user.name "ontresearch"
        - git remote add ont ${CI_REPOSITORY_URL} || true
        - git remote add github git@github.com:nanoporetech/${CI_PROJECT_NAME}.git || true
        - git fetch ont dev --tags

# Send all tags matching vX.Y.Z to github (code and release)
push-github:
    stage: release
    extends: .setup-gh-push
    script:
        # Push master and tag to github
        - git checkout ${CI_COMMIT_TAG}
        - git branch -D master || echo "No master branch to delete"
        - git fetch github
        - git checkout -b master
        - git push -f github master
        - git push github ${CI_COMMIT_TAG}
        # Make a github release page
        - python3 -m venv release_env --prompt "(release) "
        - source release_env/bin/activate
        - pip install pip --upgrade
        - pip install git+https://github.com/epi2me-labs/github_release.git
        - github_release ${CI_PROJECT_NAME} ${CI_COMMIT_TAG} CHANGELOG.md ${RESEARCH_BOT_GH_TOKEN} --artifacts dist/*.tar.gz
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/'


# Deploy to conda - need to rebuild because the template code
# is a one step build and push, for reasons...
# we override the rule so that we can push pre-release versions
.before-conda-tag: &before-conda-tag |
    export CONDA_PKG=${CI_PROJECT_NAME}
    export CONDA_PKG_VERSION=${CI_COMMIT_TAG/v/}
    cd conda

push-conda:
    extends: .deploy-conda
    variables:
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda-tag
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(\S+)?$/'

push-conda-arm:
    extends: .deploy-conda-linux-arm
    variables:
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda-tag
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(\S+)?$/'

push-conda-mac-arm:
    extends: .deploy-conda-mac-arm
    variables:
        USE_ORG_CHANNEL: "yes"
    before_script:
        - *before-conda-tag
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(\S+)?$/'