From 74f48d1aa6c4c46f2bcae77c85e006781e3c9469 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:34:18 +0100 Subject: [PATCH 01/65] [CI] First draft for ci building and testing library Signed-off-by: julian --- .github/workflows/ci.yml | 119 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..831cb6518 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,119 @@ +name: Build package on different Os and Python versions + +on : [pull_request] + +jobs: + + build_package: + name: Build ${{ github.event.repository.name }} on ${{ matrix.os }} for Python-${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + include: + - os: ubuntu-latest + path: ~/.cache/pip + # ccache requires absolute path + path_ccache: /home/runner/cache/ccache + - os: macos-latest + path: ~/Library/Caches/pip + # ccache requires absolute path + path_ccache: /Users/runner/Library/Caches/ccache + - os: windows-latest + path: ~\AppData\Local\pip\Cache + # FIXME: ccache is currently not available on Windows + # path_ccache: ~\AppData\Local\ccache\Cache + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python-${{ matrix.python-version }} + with: + python-version: ${{ matrix.python-version }} + + - name: Activating Python cache + uses: actions/cache@v2 + id: cache_python + continue-on-error: true + with: + path: ${{ matrix.path }} + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }} + + + # FIXME: This is quite costly but at the moment I don't have an alternative + - name: Install ccache for Linux and Mac + if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt -y install ccache + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install ccache + fi + shell: bash + + - name: Change name to giotto-tda-nightly + if: false + run: | + set -e + sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py + sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py + sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py + shell: bash + + # Windows does not support ccache with visual studio compiler + - name: ccache cache files + if: ${{ runner.os != 'Windows' }} + uses: actions/cache@v2 + id: cache_ccache + continue-on-error: true + with: + path: ${{ matrix.path_ccache }} + # TODO: When updating c++ backend, use new date (yy-mm-dd) + key: ${{ runner.os }}-ccache-${{ matrix.python-version }}-21-11-08 + + - name: Upgrade pip and setuptools + run: | + python -m pip install --upgrade pip setuptools + + - name: Build giotto-ph on Linux and MacOs + if: ${{ runner.os != 'Windows' }} + run: | + python -m pip install -e . ".[dev]" + env: + CC: "ccache gcc" + CXX: "ccache g++" + CCACHE_DIR: ${{ matrix.path_ccache }} + CCACHE_COMPRESS: 1 + CCACHE_MAXSIZE: 4G + + # Windows does not support ccache with visual studio compiler + - name: Build giotto-ph on Windows + if: ${{ runner.os == 'Windows' }} + run: | + python -m pip install -e . ".[dev]" + + - name: Test package with pytest + uses: actions/upload-artifact@v2 + run: | + python -m pip install pandas pytest pytest-cov pytest-azurepipelines pytest-benchmark hypothesis + mkdir coverage + cd coverage + pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload + with: + name: ${{ runner.os }}-coverage + path: coverage/ + if-no-files-found: warn + + - name: Test jupyter notebooks with papermill + run: | + python -m pip install openml matplotlib + python -m pip install papermill + cd examples + for n in *.ipynb + do + papermill --start_timeout 2000 $n - + done From faa622a2bb74117c04a711746eaf7eaf7ecb63b8 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:38:47 +0100 Subject: [PATCH 02/65] [CI] Fix upload coverage Signed-off-by: julian --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 831cb6518..fd7b9b880 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,12 +97,15 @@ jobs: python -m pip install -e . ".[dev]" - name: Test package with pytest - uses: actions/upload-artifact@v2 run: | python -m pip install pandas pytest pytest-cov pytest-azurepipelines pytest-benchmark hypothesis mkdir coverage cd coverage pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload + cd .. + + - name: Upload coverage + uses: actions/upload-artifact@v2 with: name: ${{ runner.os }}-coverage path: coverage/ From 76867cf6a929afb37c6fa3ea139962f751fb51ed Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:44:03 +0100 Subject: [PATCH 03/65] [CI] Enable temporarly on push run Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd7b9b880..aa1958450 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Build package on different Os and Python versions -on : [pull_request] +on : [push, pull_request] jobs: From 7199c1ec9dc192fbe6a724ec7bf9cb434f344957 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:46:06 +0100 Subject: [PATCH 04/65] [CI] Use repository name for jobs Signed-off-by: julian --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa1958450..dc566fd52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools - - name: Build giotto-ph on Linux and MacOs + - name: Build ${{ github.event.repository.name }} on Linux and MacOs if: ${{ runner.os != 'Windows' }} run: | python -m pip install -e . ".[dev]" @@ -91,7 +91,7 @@ jobs: CCACHE_MAXSIZE: 4G # Windows does not support ccache with visual studio compiler - - name: Build giotto-ph on Windows + - name: Build ${{ github.event.repository.name }} on Windows if: ${{ runner.os == 'Windows' }} run: | python -m pip install -e . ".[dev]" From a81f319dcb00c429c148e1405f1d65d3eee65ade Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:47:27 +0100 Subject: [PATCH 05/65] [CI] Fix python issue installation Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc566fd52..25371883c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: - name: Build ${{ github.event.repository.name }} on Linux and MacOs if: ${{ runner.os != 'Windows' }} run: | - python -m pip install -e . ".[dev]" + python -m pip install -e ".[dev]" env: CC: "ccache gcc" CXX: "ccache g++" From 57498cc207aaf574766d3ce63c230fd6c3df29d7 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:52:07 +0100 Subject: [PATCH 06/65] [CI] Add boost dependency Signed-off-by: julian --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25371883c..2e5af2952 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,15 @@ jobs: run: | python -m pip install --upgrade pip setuptools + - name: Install boost + uses: MarkusJx/install-boost@v2.1.0 + id: install-boost + with: + # REQUIRED: Specify the required boost version + # A list of supported versions can be found here: + # https://github.com/actions/boost-versions/blob/main/versions-manifest.json + boost_version: 1.76.0 + - name: Build ${{ github.event.repository.name }} on Linux and MacOs if: ${{ runner.os != 'Windows' }} run: | From 3717a247d7d7edfb1049499c32b80c8455187115 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:54:07 +0100 Subject: [PATCH 07/65] [CI] Disabling support to Python3.10 because pyflagser does not support it Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e5af2952..ce07cedad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: [3.6, 3.7, 3.8, 3.9] include: - os: ubuntu-latest path: ~/.cache/pip From 4faa5c04b73486e5efaa020ab31edf26b11a5d2c Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 00:55:23 +0100 Subject: [PATCH 08/65] [CI] Fix build on Windows Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce07cedad..77b42ff6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: - name: Build ${{ github.event.repository.name }} on Windows if: ${{ runner.os == 'Windows' }} run: | - python -m pip install -e . ".[dev]" + python -m pip install -e ".[dev]" - name: Test package with pytest run: | From 98f6a07e79bb7709a9fa05b8065ab7c2e90ff91e Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 01:02:07 +0100 Subject: [PATCH 09/65] [CI] Try to fix boost detection Signed-off-by: julian --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77b42ff6d..0fc4683c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,12 +98,15 @@ jobs: CCACHE_DIR: ${{ matrix.path_ccache }} CCACHE_COMPRESS: 1 CCACHE_MAXSIZE: 4G + BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} # Windows does not support ccache with visual studio compiler - name: Build ${{ github.event.repository.name }} on Windows if: ${{ runner.os == 'Windows' }} run: | python -m pip install -e ".[dev]" + env: + BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} - name: Test package with pytest run: | From 9457dc7a16caa5178fd520ac6f1f8af3fbf07fb9 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 01:12:21 +0100 Subject: [PATCH 10/65] [CI] Allow continue on failure if test fails Signed-off-by: julian --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fc4683c1..eee7322b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,7 @@ jobs: cd coverage pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload cd .. + continue-on-error: true - name: Upload coverage uses: actions/upload-artifact@v2 From 8fae5754703a62cb962ecabe4059f86514e7bdcb Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 01:20:00 +0100 Subject: [PATCH 11/65] [CI] Allow error with jupyter notebooks Signed-off-by: julian --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eee7322b6..84c9ef3de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,5 +131,6 @@ jobs: cd examples for n in *.ipynb do - papermill --start_timeout 2000 $n - + papermill --start_timeout 2000 $n - done + continue-on-error: true From c11db461ebcf7e6d0a68a6d4b54c1c998cebb038 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 11:53:17 +0100 Subject: [PATCH 12/65] [CI] Add coverage generation on Mac Signed-off-by: julian --- .github/workflows/ci.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84c9ef3de..2e81f5ef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,20 +108,28 @@ jobs: env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} - - name: Test package with pytest + - name: Install test tools + run: | + python -m pip install pandas pytest pytest-cov pytest-benchmark hypothesis flake8 + + - name: Test package with pytest on Linux and Windows + if: ${{ runner.os == 'Linux' || runner.os == 'Windows' }} + run: | + pytest --pyargs gtda --no-cov + + - name: Test package with pytest and flake8 on Mac with upload + if: ${{ runner.os == 'macOS' }} run: | - python -m pip install pandas pytest pytest-cov pytest-azurepipelines pytest-benchmark hypothesis mkdir coverage - cd coverage - pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload - cd .. - continue-on-error: true + pytest gtda --cov --cov-report xml:coverage.xml + flake8 - - name: Upload coverage + - name: Upload coverage for Mac + if: ${{ runner.os == 'macOS' }} uses: actions/upload-artifact@v2 with: name: ${{ runner.os }}-coverage - path: coverage/ + path: coverage.xml if-no-files-found: warn - name: Test jupyter notebooks with papermill From 3e2d8649d11fc4f7524f90d66f4b70a10e31dce2 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 11:57:57 +0100 Subject: [PATCH 13/65] [CI] Delete Azure related CI Signed-off-by: julian --- .azure-ci/build_manylinux2014.sh | 7 - .azure-ci/docker_scripts.sh | 58 ------ .azure-ci/setup_ccache.sh | 11 - azure-pipelines.yml | 342 ------------------------------- 4 files changed, 418 deletions(-) delete mode 100755 .azure-ci/build_manylinux2014.sh delete mode 100755 .azure-ci/docker_scripts.sh delete mode 100755 .azure-ci/setup_ccache.sh delete mode 100644 azure-pipelines.yml diff --git a/.azure-ci/build_manylinux2014.sh b/.azure-ci/build_manylinux2014.sh deleted file mode 100755 index 69d288230..000000000 --- a/.azure-ci/build_manylinux2014.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -docker run -t --rm -e python_ver=$PYTHON_VER \ - -v `pwd`:/io \ - -v "${CCACHE_DIR}":/root/.ccache/ \ - quay.io/pypa/manylinux2014_x86_64 \ - /bin/bash -c "bash /io/.azure-ci/docker_scripts.sh" diff --git a/.azure-ci/docker_scripts.sh b/.azure-ci/docker_scripts.sh deleted file mode 100755 index 4fc0a3a8d..000000000 --- a/.azure-ci/docker_scripts.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -e -set -x -echo "Start manylinux2014 docker build" - -# Upgrade pip and setuptools. TODO: Monitor status of pip versions -PYTHON_PATH=$(eval find "/opt/python/*cp${python_ver}*" -print) -export PATH="${PYTHON_PATH}/bin:${PATH}" -pip config set global.progress_bar off -pip install --upgrade pip setuptools - -# Install CMake -pip install cmake - -# Setup ccache -yum install -y ccache -source /io/.azure-ci/setup_ccache.sh - -ccache -s - -# Install boost -yum install -y wget tar -wget --no-check-certificate https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/boost_1_76_0.tar.gz -tar -zxvf /boost_1_76_0.tar.gz -mkdir boost -cd /boost_1_76_0 -./bootstrap.sh --prefix=/boost -./b2 install -j3 || echo "Parts of boost failed to build. Continuing..." -cd .. - -ccache -s - -# Help CMake find boost -export BOOST_ROOT=/boost -export Boost_INCLUDE_DIR=/boost/include - -# Install dev environment -git config --global --add safe.directory /io -cd /io -pip install wheel -pip install -e ".[dev]" - -# Test dev install with pytest -CI=1 # makes hypothesis print falsifying examples -pytest gtda --no-cov --no-coverage-upload - -# Uninstall giotto-tda/giotto-tda-nightly dev -pip uninstall -y giotto-tda -pip uninstall -y giotto-tda-nightly - -# Build wheels -python setup.py bdist_wheel - -# Repair wheels with auditwheel -pip install auditwheel -auditwheel repair dist/*whl -w dist/ -# remove wheels that are not manylinux2014 -rm -rf dist/*-linux*.whl diff --git a/.azure-ci/setup_ccache.sh b/.azure-ci/setup_ccache.sh deleted file mode 100755 index 661082375..000000000 --- a/.azure-ci/setup_ccache.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -echo "Setting up ccache" -mkdir /tmp/ccache/ -ln -s $(which ccache) /tmp/ccache/gcc -ln -s $(which ccache) /tmp/ccache/g++ -ln -s $(which ccache) /tmp/ccache/cc -ln -s $(which ccache) /tmp/ccache/c++ -export PATH="/tmp/ccache/:${PATH}" -# maximum cache size and compression -ccache -M 1024M -export CCACHE_COMPRESS=1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 094972319..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,342 +0,0 @@ -# These jobs are triggered automatically and they test code, examples, and wheels. -# Additional checks can be manually triggered -variables: - nightlyRelease: $[and(eq(variables['nightly_check'], true), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['Build.Reason'], 'PullRequest'))] - -trigger: -- master - -pr: -- master - -jobs: - -- job: 'manylinux2014' - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - Python37: - arch: x86_64 - plat: manylinux2014_x86_64 - python_ver: '37' - python.version: '3.7' - Python38: - arch: x86_64 - plat: manylinux2014_x86_64 - python_ver: '38' - python.version: '3.8' - Python39: - arch: x86_64 - plat: manylinux2014_x86_64 - python_ver: '39' - python.version: '3.9' - Python310: - arch: x86_64 - plat: manylinux2014_x86_64 - python_ver: '310' - python.version: '3.10' - variables: - CCACHE_DIR: $(Pipeline.Workspace)/ccache - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - - - task: Cache@2 - inputs: - key: '"ccache-wheels-v2022.8.25" | $(Agent.OS) | "$(python.version)"' - path: $(CCACHE_DIR) - displayName: ccache - - - bash: | - set -e - sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py - sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py - condition: eq(variables.nightlyRelease, true) - displayName: 'Change name to giotto-tda-nightly' - - - task: Bash@3 - inputs: - filePath: .azure-ci/build_manylinux2014.sh - env: - python_ver: $(python_ver) - CCACHE_DIR: $(CCACHE_DIR) - displayName: 'Run docker container, install and uninstall dev environment, test with pytest and flake8, build the wheels' - - - script: | - set -e - python -m pip install --upgrade pip - python -m pip install dist/*manylinux2014*.whl - displayName: 'Install the wheels' - - - script: | - set -e - python -m pip install pandas pytest pytest-cov pytest-azurepipelines pytest-benchmark hypothesis - # nbformat is needed by plotly: https://github.com/plotly/plotly.py/issues/2159 - python -m pip install nbformat - mkdir tmp_test_cov - cd tmp_test_cov - pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload - condition: eq(variables['test_wheels'], 'true') - displayName: 'Test the wheels with pytest' - - - script: | - set -e - python -m pip install openml matplotlib umap-learn gensim - python -m pip install papermill - cd examples - for n in *.ipynb - do - papermill --start_timeout 2000 $n - - done - condition: eq(variables['test_notebooks'], 'true') - displayName: 'Test jupyter notebooks with papermill' - - - task: CopyFiles@2 - displayName: 'Copy files' - inputs: - contents: 'dist/*' - targetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Create download link' - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'wheel' - - - bash: | - set -e - python -m pip install twine - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/*manylinux2014*.whl - condition: eq(variables.nightlyRelease, true) - displayName: 'Upload nightly wheels to PyPI' - - -- job: 'macOS11' - pool: - vmImage: 'macOS-11' - strategy: - matrix: - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - variables: - CCACHE_DIR: $(Pipeline.Workspace)/ccache - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - - - bash: | - set -e - sed -i.bak "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py - sed -i.bak 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py - rm setup.py.bak - sed -i.bak "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py - rm gtda/_version.py.bak - condition: eq(variables.nightlyRelease, true) - displayName: 'Change name to giotto-tda-nightly' - - - task: Cache@2 - inputs: - key: '"ccache-v2022.8.25" | $(Agent.OS) | "$(python.version)"' - path: $(CCACHE_DIR) - displayName: ccache - - # Python and gcc hotfixes following https://github.com/actions/virtual-environments/issues/2322 and https://github.com/actions/virtual-environments/issues/2391 - - script: | - set -e - brew install boost ccache - displayName: 'Install system dependencies' - - - script: | - set -e - python -m pip install --upgrade pip setuptools - python -m pip install wheel - source .azure-ci/setup_ccache.sh - python -m pip install -e ".[dev]" - ccache -s - displayName: 'Install dependencies and dev environment' - - - script: | - set -e - pytest gtda --cov --cov-report xml - displayName: 'Test dev install with pytest, upload coverage report' - - - script: | - set -e - flake8 - displayName: 'Test dev install with flake8' - - - script: | - set -e - python -m pip uninstall -y giotto-tda - python -m pip uninstall -y giotto-tda-nightly - displayName: 'Uninstall giotto-tda/giotto-tda-nightly' - - # Includes workaround for macOS 11 compatibility issue, see https://github.com/apple/tensorflow_macos/issues/46 - - script: | - set -e - python setup.py bdist_wheel - for f in dist/*.whl; do mv "$f" "${f/_11_0_/_10_16_}"; done - displayName: 'Build the wheels' - - - script: python -m pip install dist/*.whl - displayName: 'Install the wheels' - - - script: | - set -e - mkdir tmp_test_cov - cd tmp_test_cov - pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload - condition: eq(variables['test_wheels'], 'true') - displayName: 'Test the wheels with pytest' - - - script: | - set -e - python -m pip install -e ".[examples]" - python -m pip install papermill - cd examples - for n in *.ipynb - do - papermill --start_timeout 2000 $n - - done - condition: eq(variables['test_notebooks'], 'true') - displayName: 'Test jupyter notebooks with papermill' - - - task: CopyFiles@2 - displayName: 'Copy files' - inputs: - contents: 'dist/*' - targetFolder: '$(Build.ArtifactStagingDirectory)' - - - script: | - set -e - python -m pip install twine - twine check dist/* - displayName: 'Check distribution with twine' - - - task: PublishBuildArtifacts@1 - displayName: 'Create download link' - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'wheel' - - - bash: | - set -e - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/* - condition: eq(variables.nightlyRelease, true) - displayName: 'Upload nightly wheels to PyPI' - - -- job: 'windows' - pool: - vmImage: 'windows-latest' - strategy: - matrix: - Python37: - python_ver: '37' - python.version: '3.7' - Python38: - python_ver: '38' - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - - - bash: | - set -e - sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py - sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py - condition: eq(variables.nightlyRelease, true) - displayName: 'Change name to giotto-tda-nightly' - - # Use the boost_1_72_0-msvc-14.1-64.exe for windows-latest - # Following issue https://github.com/actions/virtual-environments/issues/2667 - # As of March 2021, it is necessary to download Boost manually - # DIR: installation dir, this value need to be forwarded to BOOST_ROOT_PIPELINE - - powershell: | - $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" - (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") - Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.72.0\x86_64" - displayName: 'Install boost' - - # Set BOOST_ROOT_PIPELINE to the version used in the pipeline - # See https://github.com/actions/virtual-environments/issues/687#issuecomment-616345933 - - script: | - echo "##vso[task.setvariable variable=BOOST_ROOT_PIPELINE]C:\hostedtoolcache\windows\Boost\1.72.0\x86_64" - displayName: 'Set env variable for boost version' - - - script: | - python -m pip install --upgrade pip setuptools - python -m pip install wheel - python -m pip install -e ".[dev]" - displayName: 'Install dev environment' - - - script: | - pytest gtda --no-cov --no-coverage-upload || exit /b - displayName: 'Test dev install with pytest' - - - script: | - python -m pip uninstall -y giotto-tda - python -m pip uninstall -y giotto-tda-nightly - displayName: 'Uninstall giotto-tda/giotto-tda-nightly' - - - bash: | - set -e - sed -i $'s/\r$//' README.rst - python setup.py bdist_wheel - displayName: 'Build the wheels' - - - bash: python -m pip install dist/*.whl - displayName: 'Install the wheels' - - - script: | - mkdir tmp_test_cov - cd tmp_test_cov - pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload - condition: eq(variables['test_wheels'], 'true') - displayName: 'Test the wheels with pytest' - - - script: | - python -m pip install -e ".[examples]" - python -m pip install papermill - cd examples - FOR %%n in (*.ipynb) DO (papermill --start_timeout 2000 %%n - || exit /b) - condition: eq(variables['test_notebooks'], 'true') - displayName: 'Test jupyter notebooks with papermill' - - - task: CopyFiles@2 - displayName: 'Copy files' - inputs: - contents: 'dist/*' - targetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Create download link' - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'wheel' - - - bash: | - set -e - python -m pip install twine - twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/* - condition: eq(variables.nightlyRelease, true) - displayName: 'Upload nightly wheels to PyPI' From 1e683a4bf199383660c89822386009f7feb7f511 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 12:07:47 +0100 Subject: [PATCH 14/65] [CI] flake exclude boost directory Signed-off-by: julian --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e81f5ef9..5e70761c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: with: path: ${{ matrix.path_ccache }} # TODO: When updating c++ backend, use new date (yy-mm-dd) - key: ${{ runner.os }}-ccache-${{ matrix.python-version }}-21-11-08 + key: ${{ runner.os }}-ccache-${{ matrix.python-version }}-22-01-11 - name: Upgrade pip and setuptools run: | @@ -122,7 +122,7 @@ jobs: run: | mkdir coverage pytest gtda --cov --cov-report xml:coverage.xml - flake8 + flake8 --exclude=boost - name: Upload coverage for Mac if: ${{ runner.os == 'macOS' }} From 65fb58af212d2763e84ae6cd06211fa895d05c10 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 12:08:37 +0100 Subject: [PATCH 15/65] [CI] Raise CI timeout to 15 min Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e70761c1..37f876d7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: build_package: name: Build ${{ github.event.repository.name }} on ${{ matrix.os }} for Python-${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 15 strategy: fail-fast: false matrix: From e80002fd7e42a95fb1b2df7d1bbfcde35a6e531d Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 13:49:23 +0100 Subject: [PATCH 16/65] [CI] Update flake8 use in CI Signed-off-by: julian --- .github/workflows/ci.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f876d7c..60ad9a0d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,9 +83,6 @@ jobs: uses: MarkusJx/install-boost@v2.1.0 id: install-boost with: - # REQUIRED: Specify the required boost version - # A list of supported versions can be found here: - # https://github.com/actions/boost-versions/blob/main/versions-manifest.json boost_version: 1.76.0 - name: Build ${{ github.event.repository.name }} on Linux and MacOs @@ -110,27 +107,23 @@ jobs: - name: Install test tools run: | - python -m pip install pandas pytest pytest-cov pytest-benchmark hypothesis flake8 + python -m pip install pandas pytest pytest-cov pytest-benchmark hypothesis - name: Test package with pytest on Linux and Windows if: ${{ runner.os == 'Linux' || runner.os == 'Windows' }} run: | pytest --pyargs gtda --no-cov - - name: Test package with pytest and flake8 on Mac with upload + - name: Test package with pytest Mac with coverage if: ${{ runner.os == 'macOS' }} run: | - mkdir coverage pytest gtda --cov --cov-report xml:coverage.xml - flake8 --exclude=boost - - name: Upload coverage for Mac + - name: flake8 Lint on MacOS if: ${{ runner.os == 'macOS' }} - uses: actions/upload-artifact@v2 + uses: py-actions/flake8@v2 with: - name: ${{ runner.os }}-coverage - path: coverage.xml - if-no-files-found: warn + exclude: "boost" - name: Test jupyter notebooks with papermill run: | @@ -142,3 +135,12 @@ jobs: papermill --start_timeout 2000 $n - done continue-on-error: true + + - name: Upload coverage for Mac + if: ${{ runner.os == 'macOS' }} + uses: actions/upload-artifact@v2 + with: + name: ${{ runner.os }}-coverage + path: coverage.xml + if-no-files-found: warn + From 7483f4174b826b29285024d17dadbff018da04d4 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 14:07:58 +0100 Subject: [PATCH 17/65] [CI] Update flake8 use in CI Signed-off-by: julian --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60ad9a0d3..5ee2ec43c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,7 @@ jobs: if: ${{ runner.os == 'macOS' }} uses: py-actions/flake8@v2 with: + args: "--config setup.cfg" exclude: "boost" - name: Test jupyter notebooks with papermill From c2d2817af57298eeaa3c46559bf5f526d006bfde Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 14:24:25 +0100 Subject: [PATCH 18/65] [CI] Add debug print for flake8 Signed-off-by: julian --- .github/workflows/ci.yml | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee2ec43c..701893796 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,18 @@ jobs: path: ${{ matrix.path }} key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }} + - name: Change name to giotto-tda-nightly + if: false + run: | + set -e + sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py + sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py + sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py + shell: bash + + ################### + # Caching stuff # + ################### # FIXME: This is quite costly but at the moment I don't have an alternative - name: Install ccache for Linux and Mac @@ -55,15 +67,6 @@ jobs: fi shell: bash - - name: Change name to giotto-tda-nightly - if: false - run: | - set -e - sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py - sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py - shell: bash - # Windows does not support ccache with visual studio compiler - name: ccache cache files if: ${{ runner.os != 'Windows' }} @@ -85,6 +88,11 @@ jobs: with: boost_version: 1.76.0 + + ###################### + # Building package # + ###################### + - name: Build ${{ github.event.repository.name }} on Linux and MacOs if: ${{ runner.os != 'Windows' }} run: | @@ -109,6 +117,11 @@ jobs: run: | python -m pip install pandas pytest pytest-cov pytest-benchmark hypothesis + + ############# + # Testing # + ############# + - name: Test package with pytest on Linux and Windows if: ${{ runner.os == 'Linux' || runner.os == 'Windows' }} run: | @@ -118,6 +131,8 @@ jobs: if: ${{ runner.os == 'macOS' }} run: | pytest gtda --cov --cov-report xml:coverage.xml + ls + cat setup.cfg - name: flake8 Lint on MacOS if: ${{ runner.os == 'macOS' }} @@ -137,6 +152,10 @@ jobs: done continue-on-error: true + ###################### + # Upload Artifacts # + ###################### + - name: Upload coverage for Mac if: ${{ runner.os == 'macOS' }} uses: actions/upload-artifact@v2 From b9ca33d72d4d90af87dcb7fecdadb0fd358e9f9f Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 14:41:26 +0100 Subject: [PATCH 19/65] [CI] Add hardcoded folders to exclude for dlake8 Signed-off-by: julian --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 701893796..fc40577e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,15 +131,13 @@ jobs: if: ${{ runner.os == 'macOS' }} run: | pytest gtda --cov --cov-report xml:coverage.xml - ls - cat setup.cfg + # FIXME: It does not use config.cfg, I don't know why ... - name: flake8 Lint on MacOS if: ${{ runner.os == 'macOS' }} uses: py-actions/flake8@v2 with: - args: "--config setup.cfg" - exclude: "boost" + exclude: "boost gtda/externals doc examples" - name: Test jupyter notebooks with papermill run: | From 73a97c5eb5c59295b9346b77226d2cea46e2015c Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 14:52:36 +0100 Subject: [PATCH 20/65] [CI] Add hardcoded folders to exclude for dlake8 Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc40577e8..150963f32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: if: ${{ runner.os == 'macOS' }} uses: py-actions/flake8@v2 with: - exclude: "boost gtda/externals doc examples" + exclude: "boost,gtda/externals,doc,examples" - name: Test jupyter notebooks with papermill run: | From ad95922f220e0b91efc74d76cffacfb318a3f5d8 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 15:16:18 +0100 Subject: [PATCH 21/65] [CI] Disable on push action Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 150963f32..9f54680aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Build package on different Os and Python versions -on : [push, pull_request] +on : [pull_request] jobs: From 321639b60607d60d65e2a12bed476fdad6cd9e6d Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 15:16:45 +0100 Subject: [PATCH 22/65] [CI] Add wheels generations workflow Signed-off-by: julian --- .github/workflows/wheels.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d6080b20d..24964a1c3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,5 +1,6 @@ name: Build Wheels +<<<<<<< HEAD # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_dispatchinputs on : workflow_dispatch: @@ -9,6 +10,9 @@ on : type: boolean required: true default: false +======= +on : push +>>>>>>> 902f1fd ([CI] Add wheels generations workflow) jobs: @@ -19,6 +23,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] +<<<<<<< HEAD include: - os: ubuntu-latest boost_install_dir: /home/runner/work @@ -28,10 +33,13 @@ jobs: boost_install_dir: D:\ env: BOOST_VERSION: 1.76.0 +======= +>>>>>>> 902f1fd ([CI] Add wheels generations workflow) steps: - uses: actions/checkout@v2 +<<<<<<< HEAD - name: Change name to giotto-tda-nightly if: ${{ github.event.inputs.nightlyRelease == true }} run: | @@ -90,6 +98,26 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 # Should generate universal2 wheels for CP3.8 -- CP3.10 CIBW_ARCHS_MACOS: x86_64 universal2 +======= + - name: Install boost + uses: MarkusJx/install-boost@v2.1.0 + id: install-boost + with: + boost_version: 1.76.0 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.2.2 + env: + # Specify which Python versions to build wheels + # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip + CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-*" + # Skip 32 bit architectures, musllinux, and i686 + CIBW_SKIP: "*-win32 *-musllinux_x86_64 *_i686" + CIBW_BEFORE_BUILD: python -m pip install cmake + CIBW_TEST_COMMAND: python -m pytest {package}/gph/python/test + CIBW_TEST_REQUIRES: pytest hypothesis pandas pytest-cov pytest-benchmark + BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} +>>>>>>> 902f1fd ([CI] Add wheels generations workflow) - uses: actions/upload-artifact@v2 name: Upload wheels From 11a81a01fdb4753257fd93713dd00fa6a546464a Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 21:35:12 +0100 Subject: [PATCH 23/65] [CI] Add boost install dir Signed-off-by: julian --- .github/workflows/wheels.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 24964a1c3..d6080b20d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,6 +1,5 @@ name: Build Wheels -<<<<<<< HEAD # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_dispatchinputs on : workflow_dispatch: @@ -10,9 +9,6 @@ on : type: boolean required: true default: false -======= -on : push ->>>>>>> 902f1fd ([CI] Add wheels generations workflow) jobs: @@ -23,7 +19,6 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] -<<<<<<< HEAD include: - os: ubuntu-latest boost_install_dir: /home/runner/work @@ -33,13 +28,10 @@ jobs: boost_install_dir: D:\ env: BOOST_VERSION: 1.76.0 -======= ->>>>>>> 902f1fd ([CI] Add wheels generations workflow) steps: - uses: actions/checkout@v2 -<<<<<<< HEAD - name: Change name to giotto-tda-nightly if: ${{ github.event.inputs.nightlyRelease == true }} run: | @@ -98,26 +90,6 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 # Should generate universal2 wheels for CP3.8 -- CP3.10 CIBW_ARCHS_MACOS: x86_64 universal2 -======= - - name: Install boost - uses: MarkusJx/install-boost@v2.1.0 - id: install-boost - with: - boost_version: 1.76.0 - - - name: Build wheels - uses: pypa/cibuildwheel@v2.2.2 - env: - # Specify which Python versions to build wheels - # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-*" - # Skip 32 bit architectures, musllinux, and i686 - CIBW_SKIP: "*-win32 *-musllinux_x86_64 *_i686" - CIBW_BEFORE_BUILD: python -m pip install cmake - CIBW_TEST_COMMAND: python -m pytest {package}/gph/python/test - CIBW_TEST_REQUIRES: pytest hypothesis pandas pytest-cov pytest-benchmark - BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} ->>>>>>> 902f1fd ([CI] Add wheels generations workflow) - uses: actions/upload-artifact@v2 name: Upload wheels From c17248579783a0710fb0b0afbfee3590d7c60b69 Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 23:01:25 +0100 Subject: [PATCH 24/65] [CI] Update boost logic Signed-off-by: julian --- cmake/HelperBoost.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HelperBoost.cmake b/cmake/HelperBoost.cmake index feaeff07f..6654dfc00 100644 --- a/cmake/HelperBoost.cmake +++ b/cmake/HelperBoost.cmake @@ -1,7 +1,7 @@ # Add to BOOST_ROOT variable a custom path to # ease installation of giotto-tda on Windows platforms # The custom path will be at `C:\\local\` -if(WIN32) +if(WIN32 AND $ENV{BOOST_ROOT} STREQUAL "") list(APPEND BOOST_ROOT "C:/local") list(APPEND BOOST_ROOT "") # Add custom path to your boost installation endif() From 271fc102f79e916bcaf48d0d9dc0008fcc6e0b7b Mon Sep 17 00:00:00 2001 From: julian Date: Mon, 10 Jan 2022 23:26:02 +0100 Subject: [PATCH 25/65] [CI] Revert previous change Signed-off-by: julian --- cmake/HelperBoost.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HelperBoost.cmake b/cmake/HelperBoost.cmake index 6654dfc00..feaeff07f 100644 --- a/cmake/HelperBoost.cmake +++ b/cmake/HelperBoost.cmake @@ -1,7 +1,7 @@ # Add to BOOST_ROOT variable a custom path to # ease installation of giotto-tda on Windows platforms # The custom path will be at `C:\\local\` -if(WIN32 AND $ENV{BOOST_ROOT} STREQUAL "") +if(WIN32) list(APPEND BOOST_ROOT "C:/local") list(APPEND BOOST_ROOT "") # Add custom path to your boost installation endif() From 42da492a591aa31228bcb78832e714f0b9898bba Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 14:02:32 +0100 Subject: [PATCH 26/65] [PY] Add modules in externals Signed-off-by: julian --- gtda/externals/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gtda/externals/__init__.py b/gtda/externals/__init__.py index a735ba3cb..380c21d8e 100644 --- a/gtda/externals/__init__.py +++ b/gtda/externals/__init__.py @@ -17,5 +17,6 @@ 'PeriodicCubicalComplex', 'SimplexTree', 'WitnessComplex', - 'StrongWitnessComplex' + 'StrongWitnessComplex', + 'modules' ] From 1db74b66bf47cb14241796dc8f67cabd36e25c75 Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 15:04:32 +0100 Subject: [PATCH 27/65] [TEST] Remove all __init__.py If present in test folders, pytest tries to use the repository instead of the local installation for reading the modules Signed-off-by: julian --- gtda/curves/tests/__init__.py | 0 gtda/diagrams/tests/__init__.py | 0 gtda/externals/python/tests/__init__.py | 0 gtda/graphs/tests/__init__.py | 0 gtda/homology/tests/__init__.py | 0 gtda/images/tests/__init__.py | 0 gtda/mapper/tests/__init__.py | 0 gtda/metaestimators/tests/__init__.py | 0 gtda/plotting/tests/__init__.py | 0 gtda/point_clouds/tests/__init__.py | 0 gtda/tests/__init__.py | 0 gtda/time_series/tests/__init__.py | 0 gtda/utils/tests/__init__.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 gtda/curves/tests/__init__.py delete mode 100644 gtda/diagrams/tests/__init__.py delete mode 100644 gtda/externals/python/tests/__init__.py delete mode 100644 gtda/graphs/tests/__init__.py delete mode 100644 gtda/homology/tests/__init__.py delete mode 100644 gtda/images/tests/__init__.py delete mode 100644 gtda/mapper/tests/__init__.py delete mode 100644 gtda/metaestimators/tests/__init__.py delete mode 100644 gtda/plotting/tests/__init__.py delete mode 100644 gtda/point_clouds/tests/__init__.py delete mode 100644 gtda/tests/__init__.py delete mode 100644 gtda/time_series/tests/__init__.py delete mode 100644 gtda/utils/tests/__init__.py diff --git a/gtda/curves/tests/__init__.py b/gtda/curves/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/diagrams/tests/__init__.py b/gtda/diagrams/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/externals/python/tests/__init__.py b/gtda/externals/python/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/graphs/tests/__init__.py b/gtda/graphs/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/homology/tests/__init__.py b/gtda/homology/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/images/tests/__init__.py b/gtda/images/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/mapper/tests/__init__.py b/gtda/mapper/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/metaestimators/tests/__init__.py b/gtda/metaestimators/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/plotting/tests/__init__.py b/gtda/plotting/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/point_clouds/tests/__init__.py b/gtda/point_clouds/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/tests/__init__.py b/gtda/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/time_series/tests/__init__.py b/gtda/time_series/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/gtda/utils/tests/__init__.py b/gtda/utils/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 From ee35824674f6d679c0b3ea6633be3452b005f3e0 Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 15:08:25 +0100 Subject: [PATCH 28/65] [TEST] Rename test with duplicates file names Signed-off-by: julian --- gtda/curves/tests/{test_features.py => test_curves_features.py} | 0 .../tests/{test_preprocessing.py => test_curves_preprocessing.py} | 0 .../tests/{test_distance.py => test_diagrams_distance.py} | 0 ...presentations.py => test_diagrams_features_representations.py} | 0 .../{test_preprocessing.py => test_diagrams_preprocessing.py} | 0 .../time_series/tests/{test_embedding.py => test_ts_embedding.py} | 0 gtda/time_series/tests/{test_features.py => test_ts_features.py} | 0 .../tests/{test_multivariate.py => test_ts_multivariate.py} | 0 .../tests/{test_preprocessing.py => test_ts_preprocessing.py} | 0 gtda/time_series/tests/{test_target.py => test_ts_target.py} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename gtda/curves/tests/{test_features.py => test_curves_features.py} (100%) rename gtda/curves/tests/{test_preprocessing.py => test_curves_preprocessing.py} (100%) rename gtda/diagrams/tests/{test_distance.py => test_diagrams_distance.py} (100%) rename gtda/diagrams/tests/{test_features_representations.py => test_diagrams_features_representations.py} (100%) rename gtda/diagrams/tests/{test_preprocessing.py => test_diagrams_preprocessing.py} (100%) rename gtda/time_series/tests/{test_embedding.py => test_ts_embedding.py} (100%) rename gtda/time_series/tests/{test_features.py => test_ts_features.py} (100%) rename gtda/time_series/tests/{test_multivariate.py => test_ts_multivariate.py} (100%) rename gtda/time_series/tests/{test_preprocessing.py => test_ts_preprocessing.py} (100%) rename gtda/time_series/tests/{test_target.py => test_ts_target.py} (100%) diff --git a/gtda/curves/tests/test_features.py b/gtda/curves/tests/test_curves_features.py similarity index 100% rename from gtda/curves/tests/test_features.py rename to gtda/curves/tests/test_curves_features.py diff --git a/gtda/curves/tests/test_preprocessing.py b/gtda/curves/tests/test_curves_preprocessing.py similarity index 100% rename from gtda/curves/tests/test_preprocessing.py rename to gtda/curves/tests/test_curves_preprocessing.py diff --git a/gtda/diagrams/tests/test_distance.py b/gtda/diagrams/tests/test_diagrams_distance.py similarity index 100% rename from gtda/diagrams/tests/test_distance.py rename to gtda/diagrams/tests/test_diagrams_distance.py diff --git a/gtda/diagrams/tests/test_features_representations.py b/gtda/diagrams/tests/test_diagrams_features_representations.py similarity index 100% rename from gtda/diagrams/tests/test_features_representations.py rename to gtda/diagrams/tests/test_diagrams_features_representations.py diff --git a/gtda/diagrams/tests/test_preprocessing.py b/gtda/diagrams/tests/test_diagrams_preprocessing.py similarity index 100% rename from gtda/diagrams/tests/test_preprocessing.py rename to gtda/diagrams/tests/test_diagrams_preprocessing.py diff --git a/gtda/time_series/tests/test_embedding.py b/gtda/time_series/tests/test_ts_embedding.py similarity index 100% rename from gtda/time_series/tests/test_embedding.py rename to gtda/time_series/tests/test_ts_embedding.py diff --git a/gtda/time_series/tests/test_features.py b/gtda/time_series/tests/test_ts_features.py similarity index 100% rename from gtda/time_series/tests/test_features.py rename to gtda/time_series/tests/test_ts_features.py diff --git a/gtda/time_series/tests/test_multivariate.py b/gtda/time_series/tests/test_ts_multivariate.py similarity index 100% rename from gtda/time_series/tests/test_multivariate.py rename to gtda/time_series/tests/test_ts_multivariate.py diff --git a/gtda/time_series/tests/test_preprocessing.py b/gtda/time_series/tests/test_ts_preprocessing.py similarity index 100% rename from gtda/time_series/tests/test_preprocessing.py rename to gtda/time_series/tests/test_ts_preprocessing.py diff --git a/gtda/time_series/tests/test_target.py b/gtda/time_series/tests/test_ts_target.py similarity index 100% rename from gtda/time_series/tests/test_target.py rename to gtda/time_series/tests/test_ts_target.py From ffb2efdab92fdb35eb812190596acff9ecb7e05e Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 15:09:52 +0100 Subject: [PATCH 29/65] [TEST] Update in all externals/python test to use global import Signed-off-by: julian --- gtda/externals/python/tests/test_bottleneck.py | 2 +- gtda/externals/python/tests/test_cech_complex.py | 2 +- .../python/tests/test_periodic_cubical_complex.py | 2 +- gtda/externals/python/tests/test_rips_complex.py | 10 +++++----- gtda/externals/python/tests/test_wasserstein.py | 3 ++- gtda/externals/python/tests/test_witness.py | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/gtda/externals/python/tests/test_bottleneck.py b/gtda/externals/python/tests/test_bottleneck.py index 0eb8fe589..dc9858573 100644 --- a/gtda/externals/python/tests/test_bottleneck.py +++ b/gtda/externals/python/tests/test_bottleneck.py @@ -1,4 +1,4 @@ -from ...modules.gtda_bottleneck import bottleneck_distance +from gtda.externals.modules.gtda_bottleneck import bottleneck_distance def test_trivial_empty_diagram(): diff --git a/gtda/externals/python/tests/test_cech_complex.py b/gtda/externals/python/tests/test_cech_complex.py index a81ce1aa4..bb9b42633 100644 --- a/gtda/externals/python/tests/test_cech_complex.py +++ b/gtda/externals/python/tests/test_cech_complex.py @@ -1,4 +1,4 @@ -from .. import CechComplex +from gtda.externals import CechComplex def test_minimal_cech(): diff --git a/gtda/externals/python/tests/test_periodic_cubical_complex.py b/gtda/externals/python/tests/test_periodic_cubical_complex.py index b89ee7f0b..852c42eb9 100644 --- a/gtda/externals/python/tests/test_periodic_cubical_complex.py +++ b/gtda/externals/python/tests/test_periodic_cubical_complex.py @@ -1,4 +1,4 @@ -from .. import PeriodicCubicalComplex +from gtda.externals import PeriodicCubicalComplex def test_empty_constructor(): diff --git a/gtda/externals/python/tests/test_rips_complex.py b/gtda/externals/python/tests/test_rips_complex.py index 9f344f233..ea945c872 100644 --- a/gtda/externals/python/tests/test_rips_complex.py +++ b/gtda/externals/python/tests/test_rips_complex.py @@ -1,4 +1,4 @@ -from .. import RipsComplex, SparseRipsComplex +from gtda.externals import SparseRipsComplex """Test comes from https://github.com/GUDHI/gudhi-devel/blob/master/src/python/test/test_rips_complex.py @@ -6,8 +6,8 @@ def test_empty_rips(): - rips_complex = RipsComplex() - del rips_complex + rips_complex = SparseRipsComplex() + del(rips_complex) def test_sparse_filtered_rips_from_points(): @@ -26,7 +26,7 @@ def test_sparse_filtered_rips_from_points(): def test_rips_from_points(): point_list = [[0, 0], [1, 0], [0, 1], [1, 1]] - rips_complex = RipsComplex(points=point_list, max_edge_length=42) + rips_complex = SparseRipsComplex(points=point_list, max_edge_length=42) simplex_tree = rips_complex.create_simplex_tree(max_dimension=1) @@ -64,7 +64,7 @@ def test_rips_from_points(): def test_rips_from_distance_matrix(): from math import sqrt distance_matrix = [[0], [1, 0], [1, sqrt(2), 0], [sqrt(2), 1, 1, 0]] - rips_complex = RipsComplex(distance_matrix=distance_matrix, + rips_complex = SparseRipsComplex(distance_matrix=distance_matrix, max_edge_length=42) simplex_tree = rips_complex.create_simplex_tree(max_dimension=1) diff --git a/gtda/externals/python/tests/test_wasserstein.py b/gtda/externals/python/tests/test_wasserstein.py index 5bad59ecf..755393d45 100644 --- a/gtda/externals/python/tests/test_wasserstein.py +++ b/gtda/externals/python/tests/test_wasserstein.py @@ -1,7 +1,8 @@ import math import pytest -from ...modules.gtda_wasserstein import wasserstein_distance, hera_get_infinity +from gtda.externals.modules.gtda_wasserstein import wasserstein_distance, \ + hera_get_infinity def test_trivial_empty_diagram(): diff --git a/gtda/externals/python/tests/test_witness.py b/gtda/externals/python/tests/test_witness.py index 1229fd53b..32e27e54f 100644 --- a/gtda/externals/python/tests/test_witness.py +++ b/gtda/externals/python/tests/test_witness.py @@ -1,4 +1,4 @@ -from .. import WitnessComplex, StrongWitnessComplex +from gtda.externals import WitnessComplex, StrongWitnessComplex """ Test comes from https://github.com/GUDHI/gudhi-devel/blob/master/src/python/test/test_witness_complex.py From cc3abdd2098c454585d6cba5438ba87e204fd611 Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 15:10:30 +0100 Subject: [PATCH 30/65] [EXT] Remove Gudhi RipsComplex because it is unused Signed-off-by: julian --- .../python/rips_complex_interface.py | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/gtda/externals/python/rips_complex_interface.py b/gtda/externals/python/rips_complex_interface.py index 2a3418ef2..9d9df766f 100644 --- a/gtda/externals/python/rips_complex_interface.py +++ b/gtda/externals/python/rips_complex_interface.py @@ -3,48 +3,6 @@ from . import SimplexTree -# RipsComplex python interface -class RipsComplex: - """The data structure is a one skeleton graph, or Rips graph, containing - edges when the edge length is less or equal to a given threshold. Edge - length is computed from a user given point cloud with a given distance - function, or a distance matrix. - """ - def __init__(self, points=None, distance_matrix=None, - max_edge_length=float('inf')): - """RipsComplex constructor. - :param max_edge_length: Rips value. - :type max_edge_length: float - :param points: A list of points in d-Dimension. - :type points: list of list of double - Or - :param distance_matrix: A distance matrix (full square or lower - triangular). - :type points: list of list of double - """ - self.thisref = Rips_complex_interface() - - if distance_matrix is not None: - self.thisref.init_matrix(distance_matrix, max_edge_length) - else: - if points is None: - # Empty Rips construction - points = [] - self.thisref.init_points(points, max_edge_length) - - def create_simplex_tree(self, max_dimension=1): - """ - :param max_dimension: graph expansion for rips until this given maximal - dimension. - :type max_dimension: int - :returns: A simplex tree created from the Delaunay Triangulation. - :rtype: SimplexTree - """ - simplex_tree = SimplexTree() - self.thisref.create_simplex_tree(simplex_tree.thisptr, max_dimension) - return simplex_tree - - # SparseRipsComplex python interface class SparseRipsComplex: """The data structure is a one skeleton graph, or Rips graph, containing From cb48846c4f0c9567f1442670fa41d8215acc5c22 Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 11 Jan 2022 15:22:56 +0100 Subject: [PATCH 31/65] Revert "[EXT] Remove Gudhi RipsComplex because it is unused" This reverts commit 4a75f2b570e3f44300df1e38f0fb9bf832bb6457. --- .../python/rips_complex_interface.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/gtda/externals/python/rips_complex_interface.py b/gtda/externals/python/rips_complex_interface.py index 9d9df766f..2a3418ef2 100644 --- a/gtda/externals/python/rips_complex_interface.py +++ b/gtda/externals/python/rips_complex_interface.py @@ -3,6 +3,48 @@ from . import SimplexTree +# RipsComplex python interface +class RipsComplex: + """The data structure is a one skeleton graph, or Rips graph, containing + edges when the edge length is less or equal to a given threshold. Edge + length is computed from a user given point cloud with a given distance + function, or a distance matrix. + """ + def __init__(self, points=None, distance_matrix=None, + max_edge_length=float('inf')): + """RipsComplex constructor. + :param max_edge_length: Rips value. + :type max_edge_length: float + :param points: A list of points in d-Dimension. + :type points: list of list of double + Or + :param distance_matrix: A distance matrix (full square or lower + triangular). + :type points: list of list of double + """ + self.thisref = Rips_complex_interface() + + if distance_matrix is not None: + self.thisref.init_matrix(distance_matrix, max_edge_length) + else: + if points is None: + # Empty Rips construction + points = [] + self.thisref.init_points(points, max_edge_length) + + def create_simplex_tree(self, max_dimension=1): + """ + :param max_dimension: graph expansion for rips until this given maximal + dimension. + :type max_dimension: int + :returns: A simplex tree created from the Delaunay Triangulation. + :rtype: SimplexTree + """ + simplex_tree = SimplexTree() + self.thisref.create_simplex_tree(simplex_tree.thisptr, max_dimension) + return simplex_tree + + # SparseRipsComplex python interface class SparseRipsComplex: """The data structure is a one skeleton graph, or Rips graph, containing From 86bb7b13e275772fb4213af433400da0f8e58a47 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 14:31:21 +0100 Subject: [PATCH 32/65] [CI] Disable in the wheels to download from action boost on linux worker Signed-off-by: julian --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d6080b20d..27103e592 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -42,6 +42,7 @@ jobs: shell: bash - name: Create directories boost + if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }} run: | cd ${{ matrix.boost_install_dir }} mkdir -p boost/boost From 8ae997632244f236e7cdc2728a7185a8952943bd Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 14:59:40 +0100 Subject: [PATCH 33/65] [CI] Use /host for linux docker to prevent downloading each time boost Signed-off-by: julian --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 27103e592..d6080b20d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -42,7 +42,6 @@ jobs: shell: bash - name: Create directories boost - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }} run: | cd ${{ matrix.boost_install_dir }} mkdir -p boost/boost From 6a870081f10044e639486930c3d67f2f2381c103 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:13:12 +0100 Subject: [PATCH 34/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d6080b20d..f827c64a4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,11 +21,11 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: ubuntu-latest - boost_install_dir: /home/runner/work + boost_install_dir: ${GITHUB_WORKSPACE} - os: macos-latest - boost_install_dir: /Users/runner/work + boost_install_dir: ${GITHUB_WORKSPACE} - os: windows-latest - boost_install_dir: D:\ + boost_install_dir: ${GITHUB_WORKSPACE} env: BOOST_VERSION: 1.76.0 From e4c69520819cfd1005dcddd58c5bd701c594e48e Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:19:28 +0100 Subject: [PATCH 35/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f827c64a4..95b3e51dd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,11 +21,11 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: ubuntu-latest - boost_install_dir: ${GITHUB_WORKSPACE} + boost_install_dir: ${{ GITHUB_WORKSPACE }} - os: macos-latest - boost_install_dir: ${GITHUB_WORKSPACE} + boost_install_dir: ${{ GITHUB_WORKSPACE }} - os: windows-latest - boost_install_dir: ${GITHUB_WORKSPACE} + boost_install_dir: ${{ GITHUB_WORKSPACE }} env: BOOST_VERSION: 1.76.0 From ea9b96d2a2dbc9c50712e521cb63044628d390a8 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:20:50 +0100 Subject: [PATCH 36/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 95b3e51dd..42085e0af 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -19,15 +19,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - include: - - os: ubuntu-latest - boost_install_dir: ${{ GITHUB_WORKSPACE }} - - os: macos-latest - boost_install_dir: ${{ GITHUB_WORKSPACE }} - - os: windows-latest - boost_install_dir: ${{ GITHUB_WORKSPACE }} env: BOOST_VERSION: 1.76.0 + boost_install_dir: ${{ GITHUB_WORKSPACE }} steps: - uses: actions/checkout@v2 @@ -43,7 +37,7 @@ jobs: - name: Create directories boost run: | - cd ${{ matrix.boost_install_dir }} + cd ${{ env.boost_install_dir }} mkdir -p boost/boost cd - @@ -56,7 +50,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${{ matrix.boost_install_dir }}/boost/boost + path: ${{ env.boost_install_dir }}/boost/boost # Use the version as the key to only cache the correct version key: 1-boost-${{env.BOOST_VERSION}} @@ -66,7 +60,7 @@ jobs: if: steps.cache-boost.outputs.cache-hit != 'true' with: boost_version: ${{env.BOOST_VERSION}} - boost_install_dir: ${{ matrix.boost_install_dir }} + boost_install_dir: ${{ env.boost_install_dir }} - name: Build wheels uses: pypa/cibuildwheel@v2.9.0 From 98df61e6aac383867d1ce44e2a40d51a4940d506 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:23:02 +0100 Subject: [PATCH 37/65] [CI] Remove boost_install_dir variable Signed-off-by: julian --- .github/workflows/wheels.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 42085e0af..c48eac429 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,7 +21,6 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] env: BOOST_VERSION: 1.76.0 - boost_install_dir: ${{ GITHUB_WORKSPACE }} steps: - uses: actions/checkout@v2 @@ -37,7 +36,7 @@ jobs: - name: Create directories boost run: | - cd ${{ env.boost_install_dir }} + cd ${{ GITHUB_WORKSPACE }} mkdir -p boost/boost cd - @@ -50,7 +49,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${{ env.boost_install_dir }}/boost/boost + path: ${{ GITHUB_WORKSPACE }}/boost/boost # Use the version as the key to only cache the correct version key: 1-boost-${{env.BOOST_VERSION}} @@ -60,7 +59,7 @@ jobs: if: steps.cache-boost.outputs.cache-hit != 'true' with: boost_version: ${{env.BOOST_VERSION}} - boost_install_dir: ${{ env.boost_install_dir }} + boost_install_dir: ${{ GITHUB_WORKSPACE }} - name: Build wheels uses: pypa/cibuildwheel@v2.9.0 From 65d9e5329fb0a7eb544faa40eb6fe2d61f35ba3b Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:24:49 +0100 Subject: [PATCH 38/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c48eac429..eaf43a003 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -36,7 +36,7 @@ jobs: - name: Create directories boost run: | - cd ${{ GITHUB_WORKSPACE }} + cd${GITHUB_WORKSPACE } mkdir -p boost/boost cd - @@ -49,7 +49,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${{ GITHUB_WORKSPACE }}/boost/boost + path:${GITHUB_WORKSPACE }/boost/boost # Use the version as the key to only cache the correct version key: 1-boost-${{env.BOOST_VERSION}} @@ -59,7 +59,7 @@ jobs: if: steps.cache-boost.outputs.cache-hit != 'true' with: boost_version: ${{env.BOOST_VERSION}} - boost_install_dir: ${{ GITHUB_WORKSPACE }} + boost_install_dir:${GITHUB_WORKSPACE } - name: Build wheels uses: pypa/cibuildwheel@v2.9.0 From d116e590bd1971a99e63fe3d559034f60ea0d6d9 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:25:32 +0100 Subject: [PATCH 39/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index eaf43a003..fee9e38dd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -36,7 +36,7 @@ jobs: - name: Create directories boost run: | - cd${GITHUB_WORKSPACE } + cd ${GITHUB_WORKSPACE} mkdir -p boost/boost cd - @@ -49,7 +49,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path:${GITHUB_WORKSPACE }/boost/boost + path: ${GITHUB_WORKSPACE}/boost/boost # Use the version as the key to only cache the correct version key: 1-boost-${{env.BOOST_VERSION}} @@ -59,7 +59,7 @@ jobs: if: steps.cache-boost.outputs.cache-hit != 'true' with: boost_version: ${{env.BOOST_VERSION}} - boost_install_dir:${GITHUB_WORKSPACE } + boost_install_dir: ${GITHUB_WORKSPACE} - name: Build wheels uses: pypa/cibuildwheel@v2.9.0 From 32111388bb38837a4ee50538bc7d5f6a482ad6f2 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 19:29:35 +0100 Subject: [PATCH 40/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fee9e38dd..6bdd94b47 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -19,6 +19,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + boost_install_dir: ${{ github.workspace }} + - os: macos-latest + boost_install_dir: ${{ github.workspace }} + - os: windows-latest + boost_install_dir: ${{ github.workspace }} env: BOOST_VERSION: 1.76.0 @@ -36,7 +43,7 @@ jobs: - name: Create directories boost run: | - cd ${GITHUB_WORKSPACE} + cd ${{ matrix.boost_install_dir }} mkdir -p boost/boost cd - @@ -49,7 +56,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${GITHUB_WORKSPACE}/boost/boost + path: ${{ matrix.boost_install_dir }}/boost/boost # Use the version as the key to only cache the correct version key: 1-boost-${{env.BOOST_VERSION}} @@ -59,7 +66,7 @@ jobs: if: steps.cache-boost.outputs.cache-hit != 'true' with: boost_version: ${{env.BOOST_VERSION}} - boost_install_dir: ${GITHUB_WORKSPACE} + boost_install_dir: ${{ matrix.boost_install_dir }} - name: Build wheels uses: pypa/cibuildwheel@v2.9.0 From c8386f5766eaed9b85b948b7a3534f71903aee55 Mon Sep 17 00:00:00 2001 From: julian Date: Wed, 12 Jan 2022 22:49:34 +0100 Subject: [PATCH 41/65] [CI] Partial fallback to working version Signed-off-by: julian --- .github/workflows/wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6bdd94b47..c5e969df8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,11 +21,11 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: ubuntu-latest - boost_install_dir: ${{ github.workspace }} + boost_install_dir: /home/runner/work/ - os: macos-latest - boost_install_dir: ${{ github.workspace }} + boost_install_dir: /Users/runner/work/ - os: windows-latest - boost_install_dir: ${{ github.workspace }} + boost_install_dir: D:\ env: BOOST_VERSION: 1.76.0 From e690254c57ac69d32e59df7611fe65747ccd74ff Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 00:02:31 +0100 Subject: [PATCH 42/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c5e969df8..2d8cb1b7b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,11 +21,11 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: ubuntu-latest - boost_install_dir: /home/runner/work/ + boost_install_dir: /home/runner/work - os: macos-latest - boost_install_dir: /Users/runner/work/ + boost_install_dir: /Users/runner/work - os: windows-latest - boost_install_dir: D:\ + boost_install_dir: D: env: BOOST_VERSION: 1.76.0 @@ -83,7 +83,7 @@ jobs: # nbformat is needed by plotly: https://github.com/plotly/plotly.py/issues/2159 CIBW_TEST_REQUIRES: pytest hypothesis pandas nbformat CIBW_ENVIRONMENT_WINDOWS: BOOST_ROOT='${{ matrix.boost_install_dir }}boost\boost' - CIBW_ENVIRONMENT_MACOS: BOOST_ROOT='${{ matrix.boost_install_dir }}/boost' +======IBW_ENVIRONMENT_MACOS: BOOST_ROOT='${{ matrix.boost_install_dir }}/boost' CIBW_ENVIRONMENT_LINUX: BOOST_ROOT=/host${{ matrix.boost_install_dir }}/boost CIBW_ENVIRONMENT_PASS_LINUX: BOOST_ROOT CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -vv -w {dest_dir} {wheel}" From 40541744adae855a80efcc2c71ffaa0fda6b8ab8 Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 00:04:08 +0100 Subject: [PATCH 43/65] [CI] Update boost path variables Signed-off-by: julian --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2d8cb1b7b..d6080b20d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -25,7 +25,7 @@ jobs: - os: macos-latest boost_install_dir: /Users/runner/work - os: windows-latest - boost_install_dir: D: + boost_install_dir: D:\ env: BOOST_VERSION: 1.76.0 @@ -83,7 +83,7 @@ jobs: # nbformat is needed by plotly: https://github.com/plotly/plotly.py/issues/2159 CIBW_TEST_REQUIRES: pytest hypothesis pandas nbformat CIBW_ENVIRONMENT_WINDOWS: BOOST_ROOT='${{ matrix.boost_install_dir }}boost\boost' -======IBW_ENVIRONMENT_MACOS: BOOST_ROOT='${{ matrix.boost_install_dir }}/boost' + CIBW_ENVIRONMENT_MACOS: BOOST_ROOT='${{ matrix.boost_install_dir }}/boost' CIBW_ENVIRONMENT_LINUX: BOOST_ROOT=/host${{ matrix.boost_install_dir }}/boost CIBW_ENVIRONMENT_PASS_LINUX: BOOST_ROOT CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -vv -w {dest_dir} {wheel}" From e91f0be03a5cefcead5559078ec3cc36ba11f913 Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 12:40:29 +0100 Subject: [PATCH 44/65] [DOC] Add first draft of documentation concerning wheels generation Signed-off-by: julian --- doc/contributing/ci.rst | 50 ++++++++++++++++++++++++++++++++++++++ doc/contributing/index.rst | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 doc/contributing/ci.rst diff --git a/doc/contributing/ci.rst b/doc/contributing/ci.rst new file mode 100644 index 000000000..fb831f8c8 --- /dev/null +++ b/doc/contributing/ci.rst @@ -0,0 +1,50 @@ +################ +CI Documentation +################ + +.. _ci: + +This page contains a documentation about the current github actions CI. + +.. + toctree:: + :maxdepth: 2 + :hidden: + + ci + readme_docs + +** +CI +** + +CI for Pull Request +=================== + +Pull requests +------------- + TODO + +CI for generating the wheels +============================ + +The wheel generating is a workflow that needs to be manually triggered. It can be done in the following `Link `_. The configuration file of the wheels can be found here ``.github/workflows/wheels.yml``. + +The main parts of the workflow are the following: + +* Uses `cibuildwheel `_ to generate the wheels. +* Uses `install-boost `_ to download boost. +* Uses `cache `_ cache the boost version to prevent downloading it each time the job is run. + +cibuildwheel +------------ + +For ``cibuildwheel``, some *advanced* features needed to be done, particularly about boost on Linux. The reason why Linux, required special attention is because the ``cibuildwheel`` worker run specifically on `docker images `_. Meaning that, you must provide access to the downloaded boost. As the `documentation `_ states, that a shared folder exist between the host and the docker image, located in ``/host`` for the docker image. In this folder, the entire ``/`` root folder of the host is accessible. + +install-boost +------------- + +The use of the ``install-boost`` actions, is the same as described in the `README `_. But two issues were encountered: + +1. The action failed to create the custom folder for downloading the archive. To resolve this, a previous step creates manually the folders. +2. On windows, when caching the downloaded files, the next time the job was run and the cache used. Some header files where missing. No reason was found about this behavior, and to "fix" this, on windows, we always download boost. diff --git a/doc/contributing/index.rst b/doc/contributing/index.rst index 912437ad6..07cbd7701 100644 --- a/doc/contributing/index.rst +++ b/doc/contributing/index.rst @@ -11,7 +11,7 @@ This page contains a summary of what one needs to do to contribute. :hidden: guidelines - readme_docs + ci ********** Guidelines From f06398f4f5ea30d852cf1d3b1cd01424432402be Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 13:35:44 +0100 Subject: [PATCH 45/65] [CI] Fix notebook test Signed-off-by: julian --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f54680aa..dd43cfc79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,11 +144,19 @@ jobs: python -m pip install openml matplotlib python -m pip install papermill cd examples - for n in *.ipynb + # Get all .ipynb notebook but discard the MNIST_classification + # Because it is too long + # This command also removes the .pynb extension + FILES_WITHOUT_EXT=`find . ! -name '*MNIST_classification*' -iname '*.ipynb' -exec bash -c 'printf "%s\n" "${@%.*}"' _ {} +` + PAPERMILL_DIR=papermill_results + + mkdir -p $PAPERMILL_DIR + + for FILENAME in $FILES_WITHOUT_EXT do - papermill --start_timeout 2000 $n - + SAVE_FILE=$PAPERMILL_DIR/$FILENAME.txt + papermill --execution-timeout 60 $FILENAME.ipynb $SAVE_FILE done - continue-on-error: true ###################### # Upload Artifacts # From d135df6c4f57078168a771a7ff0c0c8e064a13ab Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 13:36:32 +0100 Subject: [PATCH 46/65] [CI] Enable again on push for ci Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd43cfc79..e7eab4036 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Build package on different Os and Python versions -on : [pull_request] +on : [pull_request, push] jobs: From be5041243b52b8973ac038c586c1eb64dd91292e Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 14:00:10 +0100 Subject: [PATCH 47/65] [CI] Update timeout and specify shell Signed-off-by: julian --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7eab4036..ae47c7f19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,8 +155,9 @@ jobs: for FILENAME in $FILES_WITHOUT_EXT do SAVE_FILE=$PAPERMILL_DIR/$FILENAME.txt - papermill --execution-timeout 60 $FILENAME.ipynb $SAVE_FILE + papermill --execution-timeout 120 $FILENAME.ipynb $SAVE_FILE done + shell: bash ###################### # Upload Artifacts # From 8079510bcda79218b763d3ce857768f28c26ef9a Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 22:57:08 +0100 Subject: [PATCH 48/65] [CI] Update timeout for papermill job Signed-off-by: julian --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae47c7f19..c174902fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,14 +16,17 @@ jobs: include: - os: ubuntu-latest path: ~/.cache/pip + papermill_timeout: 120 # ccache requires absolute path path_ccache: /home/runner/cache/ccache - os: macos-latest path: ~/Library/Caches/pip + papermill_timeout: 120 # ccache requires absolute path path_ccache: /Users/runner/Library/Caches/ccache - os: windows-latest path: ~\AppData\Local\pip\Cache + papermill_timeout: 240 # FIXME: ccache is currently not available on Windows # path_ccache: ~\AppData\Local\ccache\Cache @@ -155,7 +158,7 @@ jobs: for FILENAME in $FILES_WITHOUT_EXT do SAVE_FILE=$PAPERMILL_DIR/$FILENAME.txt - papermill --execution-timeout 120 $FILENAME.ipynb $SAVE_FILE + papermill --execution-timeout matrix.papermill_timeout $FILENAME.ipynb $SAVE_FILE done shell: bash From 2efab19bcc78fc5fba642941c8b0758c3b5842fa Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 23:07:28 +0100 Subject: [PATCH 49/65] [CI] Update timeout for papermill job Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c174902fe..5b0706d00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,7 +158,7 @@ jobs: for FILENAME in $FILES_WITHOUT_EXT do SAVE_FILE=$PAPERMILL_DIR/$FILENAME.txt - papermill --execution-timeout matrix.papermill_timeout $FILENAME.ipynb $SAVE_FILE + papermill --execution-timeout ${{ matrix.papermill_timeout }} $FILENAME.ipynb $SAVE_FILE done shell: bash From 6de8f2987b65d7329c952d23d4f87b6403ef4aa9 Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 23:25:42 +0100 Subject: [PATCH 50/65] [CI] Update timeout for papermill job Signed-off-by: julian --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b0706d00..196a30583 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,12 +21,12 @@ jobs: path_ccache: /home/runner/cache/ccache - os: macos-latest path: ~/Library/Caches/pip - papermill_timeout: 120 + papermill_timeout: 240 # ccache requires absolute path path_ccache: /Users/runner/Library/Caches/ccache - os: windows-latest path: ~\AppData\Local\pip\Cache - papermill_timeout: 240 + papermill_timeout: 300 # FIXME: ccache is currently not available on Windows # path_ccache: ~\AppData\Local\ccache\Cache From 561355141588565ca6e9d5866b9242c88e3cd88b Mon Sep 17 00:00:00 2001 From: julian Date: Thu, 13 Jan 2022 23:46:37 +0100 Subject: [PATCH 51/65] [CI] Update main timeout to 20 min Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 196a30583..5a04f1fc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: build_package: name: Build ${{ github.event.repository.name }} on ${{ matrix.os }} for Python-${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: From de4aaf875a6b9b8921db04a851cba55d69e5e2aa Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 16:37:38 +0100 Subject: [PATCH 52/65] [CI] Disable on push trigger Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a04f1fc6..a3f4e19dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Build package on different Os and Python versions -on : [pull_request, push] +on : [pull_request] jobs: From 838ba2748247927cbc66be7038ded986dfa678d4 Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 16:42:42 +0100 Subject: [PATCH 53/65] [CI] Remove nightly build related stuff Signed-off-by: julian --- .github/workflows/ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3f4e19dc..1625b5270 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,15 +46,6 @@ jobs: path: ${{ matrix.path }} key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }} - - name: Change name to giotto-tda-nightly - if: false - run: | - set -e - sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py - sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py - sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py - shell: bash - ################### # Caching stuff # ################### From bc301f6e7938fca057d423495d3176f22929be5a Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:10:43 +0100 Subject: [PATCH 54/65] [CI] Add boost caching Signed-off-by: julian --- .github/workflows/ci.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1625b5270..6df79d56f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,8 @@ jobs: papermill_timeout: 300 # FIXME: ccache is currently not available on Windows # path_ccache: ~\AppData\Local\ccache\Cache + env: + BOOST_VERSION: 1.76.0 steps: - uses: actions/checkout@v2 @@ -38,6 +40,10 @@ jobs: with: python-version: ${{ matrix.python-version }} + ################### + # Caching stuff # + ################### + - name: Activating Python cache uses: actions/cache@v2 id: cache_python @@ -46,9 +52,18 @@ jobs: path: ${{ matrix.path }} key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }} - ################### - # Caching stuff # - ################### + - name: Cache boost ${{env.BOOST_VERSION}} + # FIXME For an unknow reason on windows runner, when the cache is hit + # The compilation fails because it doesn't find some headers + # I don't know if the cache is corrupted, for the moment hard to debug + if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + uses: actions/cache@v2 + id: cache-boost + with: + # Set the default path as the path to cache + path: ${{env.GITHUB_WORKSPACE}}/boost/boost + # Use the version as the key to only cache the correct version + key: boost-${{env.BOOST_VERSION}} # FIXME: This is quite costly but at the moment I don't have an alternative - name: Install ccache for Linux and Mac @@ -76,11 +91,12 @@ jobs: run: | python -m pip install --upgrade pip setuptools - - name: Install boost + - name: Install boost ${{env.BOOST_VERSION}} uses: MarkusJx/install-boost@v2.1.0 id: install-boost + if: steps.cache-boost.outputs.cache-hit != 'true' with: - boost_version: 1.76.0 + boost_version: ${{env.BOOST_VERSION}} ###################### From 7c07c28f5ad1370613944b28d81ad4102ba01981 Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:11:14 +0100 Subject: [PATCH 55/65] [CI] Add input variable to enable notebook verification Signed-off-by: julian --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6df79d56f..367145c38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,14 @@ name: Build package on different Os and Python versions -on : [pull_request] +on : + pull_request: + workflow: + inputs: + testNotebooks: + description: 'Enable notebook verification step' + type: boolean + required: false + default: false jobs: @@ -150,6 +158,7 @@ jobs: exclude: "boost,gtda/externals,doc,examples" - name: Test jupyter notebooks with papermill + if: ${{ github.event.inputs.testNotebooks == true }} run: | python -m pip install openml matplotlib python -m pip install papermill From 787fb9414ef107021b48870ba491f5888e56da5e Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:12:02 +0100 Subject: [PATCH 56/65] [CI] Enable again on push for CI Signed-off-by: julian --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 367145c38..4c1855295 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on : type: boolean required: false default: false + push: jobs: From ae44ec544690ad20b1a0e3ef098d84f25de25b1c Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:13:00 +0100 Subject: [PATCH 57/65] [CI] Fix wrong name of event Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c1855295..599a9061c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: Build package on different Os and Python versions on : pull_request: - workflow: + workflow_dispatch: inputs: testNotebooks: description: 'Enable notebook verification step' From a7731eaadf41bf2e21da10da48f00efd075adf1f Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:33:06 +0100 Subject: [PATCH 58/65] [CI] Hardcode boost directory, because when cache is used, the variable is not set Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 599a9061c..e061a1df4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: CCACHE_DIR: ${{ matrix.path_ccache }} CCACHE_COMPRESS: 1 CCACHE_MAXSIZE: 4G - BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} + BOOST_ROOT: ${{env.GITHUB_WORKSPACE}}/boost/boost # Windows does not support ccache with visual studio compiler - name: Build ${{ github.event.repository.name }} on Windows From ac48e063c07056747a9a1cac430b8c9a3ab6ff6a Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:37:34 +0100 Subject: [PATCH 59/65] [CI] Add debug information Signed-off-by: julian --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e061a1df4..718f2e588 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,7 @@ jobs: - name: Build ${{ github.event.repository.name }} on Linux and MacOs if: ${{ runner.os != 'Windows' }} run: | + echo $BOOST_ROOT python -m pip install -e ".[dev]" env: CC: "ccache gcc" From 1f384bb689795fdcb01f149045ac312d212f7f30 Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:48:42 +0100 Subject: [PATCH 60/65] [CI] Add debug information Signed-off-by: julian --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 718f2e588..e33b7c716 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${{env.GITHUB_WORKSPACE}}/boost/boost + path: ${ env.GITHUB_WORKSPACE }/boost/boost # Use the version as the key to only cache the correct version key: boost-${{env.BOOST_VERSION}} @@ -123,7 +123,7 @@ jobs: CCACHE_DIR: ${{ matrix.path_ccache }} CCACHE_COMPRESS: 1 CCACHE_MAXSIZE: 4G - BOOST_ROOT: ${{env.GITHUB_WORKSPACE}}/boost/boost + BOOST_ROOT: ${ env.GITHUB_WORKSPACE }/boost/boost # Windows does not support ccache with visual studio compiler - name: Build ${{ github.event.repository.name }} on Windows From c11696b85bd3e6e7ae9b41c32d3ace14b7a24468 Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 17:51:49 +0100 Subject: [PATCH 61/65] [CI] Add debug information Signed-off-by: julian --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e33b7c716..a877b3ae9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: id: cache-boost with: # Set the default path as the path to cache - path: ${ env.GITHUB_WORKSPACE }/boost/boost + path: ${{ github.workspace }}/boost/boost # Use the version as the key to only cache the correct version key: boost-${{env.BOOST_VERSION}} @@ -123,7 +123,7 @@ jobs: CCACHE_DIR: ${{ matrix.path_ccache }} CCACHE_COMPRESS: 1 CCACHE_MAXSIZE: 4G - BOOST_ROOT: ${ env.GITHUB_WORKSPACE }/boost/boost + BOOST_ROOT: ${{ github.workspace }}/boost/boost # Windows does not support ccache with visual studio compiler - name: Build ${{ github.event.repository.name }} on Windows From c84e419334003eae91bd8fbf647c4f3701027d8e Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 19:19:55 +0100 Subject: [PATCH 62/65] [DOC] Add first draft of documentation related to the CI Signed-off-by: julian --- doc/contributing/ci.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/contributing/ci.rst b/doc/contributing/ci.rst index fb831f8c8..5814572e7 100644 --- a/doc/contributing/ci.rst +++ b/doc/contributing/ci.rst @@ -23,7 +23,22 @@ CI for Pull Request Pull requests ------------- - TODO + +On pull request the CI will be automatically triggered for each new push of commits that are done in the PR. This workflow can also be manually triggered for testing the notebooks. By default the test are disable because, notebook verification with ``papermill`` is time consuming. + +The workflow to build and validate the new PR is relatively big, it is decomposed in sections: + +* Setting up repository and the Python version +* Setting up and retrieving when available data in caches +* Install requirements and build ``giotto-tda`` library +* Install test requirements and test the compiled library +* Upload artifacts + +There are some steps that are performed only on certain platforms, the relevant ones are: + +* On windows, caching the boost library is disable, see ``ìnstall-boost section in Wheels generation``. +* Building the library for Linux and Mac is done on a different step, because they use ``ccache`` for caching intermediate files. It is not available on Windows. +* We generate coverage report and test with ``flake8`` on Mac. CI for generating the wheels ============================ From 4e3a9ce17203a4c4cefe7f4b7de8f1f60aaef471 Mon Sep 17 00:00:00 2001 From: julian Date: Sun, 16 Jan 2022 19:20:32 +0100 Subject: [PATCH 63/65] [CI] Disable on push event Signed-off-by: julian --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a877b3ae9..734aeb18f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,6 @@ on : type: boolean required: false default: false - push: jobs: From d83615769b0baaf717ea676ec00fc1d4a1db7d3a Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 30 Aug 2022 22:34:37 +0200 Subject: [PATCH 64/65] [CI] Remove 3.6 enable 3.10 Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 734aeb18f..0df39cbc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, 3.10] include: - os: ubuntu-latest path: ~/.cache/pip From a014bc926165c8d9e0f91ec4ac131fa630e37d82 Mon Sep 17 00:00:00 2001 From: julian Date: Tue, 30 Aug 2022 22:46:51 +0200 Subject: [PATCH 65/65] [CI] Remove 3.6 enable 3.10 Signed-off-by: julian --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0df39cbc8..ff47fb9b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, '3.10'] include: - os: ubuntu-latest path: ~/.cache/pip