do debugging check for POOCH_CACHE one more time #2901
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build PyPop | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| - '**.rst' | |
| - 'CITATION.cff' | |
| - '.zenodo.json' | |
| - 'website/**' | |
| - 'data/**' | |
| - 'src/obsolete/**' | |
| - '.github/dependabot.yml' | |
| - '.github/release-drafter.yml' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| - '.github/workflows/documentation.yaml' | |
| - '.github/workflows/buildjet_arm64.yml' | |
| - '.github/workflows/release-drafter.yml' | |
| - '.github/workflows/codeql.yml' | |
| - '.github/workflows/nuget_gsl_arm64_package.yml' | |
| - '.gitattributes' | |
| - '.pre-commit-config.yaml' | |
| push: | |
| paths-ignore: | |
| - '**.md' | |
| - '**.rst' | |
| - 'CITATION.cff' | |
| - '.zenodo.json' | |
| - 'website/**' | |
| - 'data/**' | |
| - 'src/obsolete/**' | |
| - '.github/dependabot.yml' | |
| - '.github/release-drafter.yml' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| - '.github/workflows/documentation.yaml' | |
| - '.github/workflows/buildjet_arm64.yml' | |
| - '.github/workflows/release-drafter.yml' | |
| - '.github/workflows/codeql.yml' | |
| - '.github/workflows/nuget_gsl_arm64_package.yml' | |
| - '.gitattributes' | |
| - '.pre-commit-config.yaml' | |
| release: | |
| types: | |
| - published | |
| env: | |
| # set MSF version and cache directory name globally | |
| MSF_VERSION: 3.59.0-alpha | |
| MSF_CACHE_DIR: msf-files-3.59.0-alpha | |
| jobs: | |
| skip_publish_zenodo: | |
| name: Non publication, Zenodo skip | |
| runs-on: ubuntu-latest | |
| # FIXME: this is the default action, the logical inverse of the | |
| # below "if" in publish_zenodo: a bit hacky, but it works | |
| if: (github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false) == false | |
| steps: | |
| - name: Skip Zenodo step unless publishing release | |
| run: | | |
| echo "Not a publication release, skipping Zenodo" | |
| echo "github.ref: ${{ toJson(github.ref) }}" | |
| echo "github.event_name: ${{ toJson(github.event_name) }}" | |
| echo "github.event.action: ${{ toJson(github.event.action) }}" | |
| publish_zenodo: | |
| name: Production release, Zenodo publication | |
| runs-on: ubuntu-latest | |
| # "else" we *are* publishing something, only works for non-pre-releases on the main branch | |
| if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false | |
| steps: | |
| - name: Checkout the contents of your repository | |
| uses: actions/checkout@v4 | |
| - name: Upsert the version and related ids in the metadata in .zenodo.json | |
| run: | | |
| echo "GITHUB_REF_NAME = $GITHUB_REF_NAME | GITHUB_REF = $GITHUB_REF | GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH" | |
| echo "github.event.release.target_commitish = ${{ github.event.release.target_commitish }}" | |
| # convert semver to Python-style version | |
| VERSION_PYTHON_STYLE=$(GITHUB_REF_NAME=$GITHUB_REF_NAME python -c 'import os; from packaging import version; print(str(version.Version(os.getenv("GITHUB_REF_NAME"))))') | |
| echo "VERSION_PYTHON_STYLE = $VERSION_PYTHON_STYLE" | |
| # cp .zenodo.json .zenodo.orig.json | |
| # update the 'related_identifiers' to use exact tree and pypi package | |
| cat .zenodo.json | jq "(.related_identifiers[] | (select (.relation == \"isSourceOf\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://pypi.org/project/pypop-genomics\")).identifier) |= \"https://pypi.org/project/pypop-genomics/$VERSION_PYTHON_STYLE\"" | jq "(.related_identifiers[] | (select (.relation == \"isSupplementTo\" and .resource_type == \"software\")) | select (.identifier | startswith(\"https://github.com/alexlancaster/pypop\")).identifier) |= \"https://github.com/alexlancaster/pypop/tree/$GITHUB_REF_NAME\"" > .zenodo.mod.json | |
| # update the version number | |
| echo "{\"version\": \"$GITHUB_REF_NAME\"}" | cat .zenodo.mod.json - | jq -s add > .zenodo.json | |
| # debugging only | |
| cat .zenodo.json | |
| rm .zenodo.mod.json | |
| - name: Upsert the version in the metadata in CITATION.cff | |
| run: | | |
| yq -i ".version = \"$GITHUB_REF_NAME\"" CITATION.cff | |
| - name: Push changes back to repo files | |
| run: | | |
| git fetch origin | |
| git config --global user.name "ci-pypop" | |
| git config --global user.email "username@users.noreply.github.com" | |
| git checkout -b $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
| git add .zenodo.json CITATION.cff | |
| git commit -m "Upserted version into CITATION.cff and into .zenodo.json" | |
| git checkout ${{ github.event.release.target_commitish }} | |
| git merge $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
| git push origin ${{ github.event.release.target_commitish }} | |
| # FIXME: removing files marked with 'export-ignore' 'set' in .gitattributes | |
| # before zenodraft creates the zipball using https://stackoverflow.com/a/30573319 | |
| # to ensure that we handle filenames with potential ':' or newlines | |
| git ls-files -z|git check-attr --stdin -z export-ignore |sed -zne 'x;n;n;s/^set$//;t print;b;:print;x;p'|xargs --null rm | |
| # FIXME: need to also remove orphaned empty directories resulting from above | |
| find . -empty -type d -delete | |
| - name: Create a draft snapshot of your repository contents as a new | |
| version in test collection on Zenodo using metadata | |
| from repository file .zenodo.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} | |
| uses: zenodraft/action@0.13.3 | |
| # uses: alexlancaster/zenodraft-action@0.10.2 | |
| with: | |
| concept: 10080667 | |
| metadata: .zenodo.json | |
| publish: false | |
| sandbox: false | |
| compression: zip | |
| upsert-doi: true | |
| upsert-location: doi | |
| verbose: true | |
| generate-wheels-matrix: | |
| # Create a matrix of all architectures & versions to build. | |
| # This enables the next step to run cibuildwheel in parallel. | |
| # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 | |
| name: Generate wheels matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| include: ${{ steps.set-matrix.outputs.include }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install cibuildwheel | |
| # Nb. keep cibuildwheel version pin consistent with job below | |
| run: pipx install cibuildwheel==2.22.0 | |
| - id: set-matrix | |
| run: | | |
| echo "CI_ONLY" $CI_ONLY | |
| if [ "$CI_ONLY" == "true" ]; then | |
| MATRIX="[{'only':'cp313-manylinux_x86_64','os':'ubuntu-20.04'},{'only':'cp313-manylinux_aarch64','os':'ubuntu-22.04-arm'},{'only':'cp313-win_amd64','os':'windows-2019'},{'only':'cp313-win_arm64','os':'windows-2019'},{'only':'cp313-macosx_x86_64','os':'macos-13'}, {'only':'cp313-macosx_arm64','os':'macos-14'}]" | |
| else | |
| MATRIX=$( | |
| { | |
| cibuildwheel --print-build-identifiers --platform linux --archs x86_64 \ | |
| | jq -nRc '{"only": inputs, "os": "ubuntu-20.04"}' \ | |
| && cibuildwheel --print-build-identifiers --platform linux --archs aarch64 \ | |
| | jq -nRc '{"only": inputs, "os": "ubuntu-22.04-arm"}' \ | |
| && cibuildwheel --print-build-identifiers --platform macos --archs x86_64 \ | |
| | jq -nRc '{"only": inputs, "os": "macos-13"}' \ | |
| && cibuildwheel --print-build-identifiers --platform macos --archs arm64 \ | |
| | jq -nRc '{"only": inputs, "os": "macos-14"}' \ | |
| && cibuildwheel --print-build-identifiers --platform windows \ | |
| | jq -nRc '{"only": inputs, "os": "windows-2019"}' | |
| } | jq -sc | |
| ) | |
| fi | |
| echo "include=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "platform matrix:" $MATRIX | |
| env: | |
| # only generate all wheels on releases, pull requests or commits to the main branch | |
| # otherwise just generate a test wheel Python 3.12 for 4 platform | |
| CI_ONLY: ${{ github.event_name != 'release' && github.event_name != 'pull_request' && github.ref_name != 'main' && github.event_name != 'workflow_dispatch' }} | |
| # set up a shared cache for downloading data files (e.g. MSF files) | |
| # before matrix jobs | |
| setup-cache: | |
| name: Setup cache for remote files | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up cache for MSF files | |
| id: msf_cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.MSF_CACHE_DIR }} | |
| key: pooch-cache-${{ env.MSF_CACHE_DIR }} | |
| restore-keys: | | |
| pooch-cache-${{ env.MSF_CACHE_DIR }} | |
| enableCrossOsArchive: true | |
| fail-on-cache-miss: false | |
| - name: Preload MSF files into Cache | |
| if: ${{ steps.msf_cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| echo "Cache not found or is outdated. Fetching MSF files..." | |
| pip install pooch requests | |
| python -c " | |
| import os | |
| import pooch | |
| import requests | |
| # Define the base URL and path for the MSF files | |
| base_url = 'https://raw.githubusercontent.com/ANHIG/IMGTHLA/v${MSF_VERSION}/msf/' | |
| # create cache directory | |
| cache_dir = '${MSF_CACHE_DIR}' | |
| # fetch the directory listing of MSF files from GitHub (raw listing) | |
| response = requests.get('https://api.github.com/repos/ANHIG/IMGTHLA/contents/msf?ref=v${MSF_VERSION}') | |
| files = response.json() | |
| # extract only prot.msf and nuc.msf files from the response | |
| msf_files = [file['name'] for file in files if file['name'].endswith('_prot.msf') or file['name'].endswith('_nuc.msf')] | |
| # fetch each MSF file using pooch.retrieve | |
| for file in msf_files: | |
| print(f'Fetching {file}...') | |
| local_file = pooch.retrieve( | |
| url=f'{base_url}{file}', # full URL to the file | |
| known_hash=None, # pooch will calculate the hash automatically | |
| path=cache_dir # cache location | |
| ) | |
| print(f'Saved {local_file}...') | |
| print('All MSF files fetched and cached successfully.') | |
| " | |
| - name: Save the cache for MSF files | |
| if: ${{ steps.msf_cache.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.MSF_CACHE_DIR }} | |
| key: pooch-cache-${{ env.MSF_CACHE_DIR }} | |
| enableCrossOsArchive: true | |
| build_wheels: | |
| name: Build wheels on ${{ matrix.only }} | |
| # depend on zenodo, to make sure updated CITATION.cff gets into builds | |
| # this only results in changes upon production releases | |
| needs: [publish_zenodo, skip_publish_zenodo, generate-wheels-matrix, setup-cache] | |
| # trying solution here: https://github.com/actions/runner/issues/491#issuecomment-1507495166 | |
| # so that it works if either job in "needs" completes | |
| if: always() && !cancelled() && !failure() | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
| ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
| - name: Restore cache for MSF files | |
| uses: actions/cache/restore@v4 | |
| with: | |
| # restore to the workspace, not home directory | |
| path: ${{ env.MSF_CACHE_DIR }} | |
| key: pooch-cache-${{ env.MSF_CACHE_DIR }} | |
| restore-keys: | | |
| pooch-cache-${{ env.MSF_CACHE_DIR }} | |
| enableCrossOsArchive: true | |
| - name: Do cache setup based on OS | |
| shell: bash | |
| run: | | |
| echo "Setting POOCH_CACHE based on runner OS" | |
| if [[ "$RUNNER_OS" == "Linux" ]]; then | |
| # adding /project to path so it works inside cibuildwheel containers | |
| POOCH_CACHE="/project/${MSF_CACHE_DIR}" | |
| else | |
| POOCH_CACHE="${GITHUB_WORKSPACE}/${MSF_CACHE_DIR}" | |
| fi | |
| echo "POOCH_CACHE=${POOCH_CACHE}" | |
| echo "POOCH_CACHE=${POOCH_CACHE}" >> $GITHUB_ENV | |
| - name: Install a recent stable Python to handle Python deps | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: Query version with setuptools_scm | |
| id: version | |
| shell: bash | |
| run: | | |
| python -m pip install setuptools_scm | |
| python -m pip install toml | |
| VERSION_SCHEME=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['tool']['setuptools_scm']['version_scheme'])") | |
| VERSION=$(python -c "import setuptools_scm; print(setuptools_scm.get_version(version_scheme=\"${VERSION_SCHEME}\"))") | |
| echo "VERSION=${VERSION}" | |
| echo "VERSION=${VERSION}" >> $GITHUB_ENV | |
| - name: Install toml and remove cffconvert from pyproject.toml | |
| run: | | |
| python -c " | |
| import toml | |
| with open('pyproject.toml', 'r') as f: | |
| config = toml.load(f) | |
| if 'build-system' in config and 'requires' in config['build-system']: | |
| config['build-system']['requires'] = [ | |
| dep for dep in config['build-system']['requires'] if 'cffconvert' not in dep.lower() | |
| ] | |
| with open('pyproject.toml', 'w') as f: | |
| toml.dump(config, f) | |
| " | |
| - name: Generate citation formats | |
| run: | | |
| python --version | |
| python -m pip install git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert | |
| python src/PyPop/citation.py | |
| - name: Build and test wheels | |
| uses: pypa/cibuildwheel@v2.22.0 | |
| env: | |
| # FIXME: only run the slow tests when doing regular pushes, or manual - not for PRs | |
| CIBW_TEST_COMMAND: "pytest -s -v {package}/tests ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '--runslow' || '' }}" | |
| SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.VERSION }} | |
| POOCH_CACHE: ${{ env.POOCH_CACHE }} | |
| CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION POOCH_CACHE | |
| with: | |
| only: ${{ matrix.only }} | |
| package-dir: . | |
| output-dir: wheelhouse | |
| config-file: "{package}/pyproject.toml" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.only }} | |
| path: ./wheelhouse/*.whl | |
| # path: ./wheelhouse/* use if more than wheels are wanted | |
| build_sdist: | |
| name: Build source distribution | |
| # depend on zenodo, to make sure updated CITATION.cff gets into builds | |
| needs: [publish_zenodo, skip_publish_zenodo] | |
| if: always() && !cancelled() && !failure() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
| ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
| fetch-depth: 0 | |
| - name: Build sdist | |
| run: pipx run build --sdist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| upload_gh_release: | |
| name: Upload binary wheels and sdist to GH release page | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| # alternatively, to publish when a GitHub Release is created, use the following rule: | |
| if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - uses: softprops/action-gh-release@v2 | |
| name: Uploading binaries to release page | |
| with: | |
| files: dist/* | |
| upload_test_pypi: | |
| name: Upload to Test_PyPI | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| environment: test_pypi | |
| permissions: | |
| id-token: write | |
| # drop requirement of 'main', release to test_pypi to test releases | |
| if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository_url: https://test.pypi.org/legacy/ | |
| upload_pypi: | |
| name: Upload to PyPI | |
| needs: [build_wheels, build_sdist] | |
| runs-on: ubuntu-latest | |
| environment: pypi | |
| permissions: | |
| id-token: write | |
| if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| # unpacks default artifact into dist/ | |
| path: dist | |
| merge-multiple: true | |
| - name: Exclude win_arm64 wheels | |
| run: | | |
| find dist -name '*win_arm64*.whl' -exec rm {} + | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 |