ENH: Enable output datasets to store checkpoints #1778

Workflow file for this run

.github/workflows/cpath-pr.yml at 029d0fc

	name: Pathology PR
	on:
	push:
	branches: [ main ]
	tags:
	- '*'
	pull_request:
	branches: [ main ]
	paths:
	- "hi-ml-cpath/**"
	- ".github/workflows/cpath-pr.yml"
	# When the core hi-ml libraries change, test the histo test too, to find problems with backwards compatibility
	- "hi-ml-azure/**"
	- "hi-ml/**"

	# Cancel previous runs of this workflow that are still in progress.
	concurrency:
	group: ${{ github.ref }}/cpath-pr
	cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

	env:
	pythonVersion: 3.9
	folder: hi-ml-cpath
	HIML_TENANT_ID: ${{ secrets.HIML_TENANT_ID }}
	HIML_RESOURCE_GROUP: ${{ secrets.HIML_RESOURCE_GROUP }}
	HIML_SUBSCRIPTION_ID: ${{ secrets.HIML_SUBSCRIPTION_ID }}
	HIML_WORKSPACE_NAME: ${{ secrets.HIML_WORKSPACE_NAME }}
	HIML_SERVICE_PRINCIPAL_ID: ${{ secrets.HIML_SERVICE_PRINCIPAL_ID }}
	HIML_SERVICE_PRINCIPAL_PASSWORD: ${{ secrets.HIML_SERVICE_PRINCIPAL_PASSWORD }}
	# Set the AML experiment name for all AML jobs submitted during tests. Github.ref looks like
	# "refs/pull/123/merge" for PR builds.
	HIML_EXPERIMENT_NAME: ${{ github.ref }}

	jobs:

	cancel-azureml:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3

	- name: Cancel previous AzureML runs
	uses: ./.github/actions/cancel_azureml_jobs

	cpath-flake8:
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Set up Python ${{ env.pythonVersion }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.pythonVersion }}

	- name: flake8
	run: \|
	cd ${{ env.folder }}
	make pip_test
	make flake8

	cpath-mypy:
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Set up Python ${{ env.pythonVersion }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.pythonVersion }}

	- name: mypy
	run: \|
	cd ${{ env.folder }}
	make pip_test
	make mypy

	cpath-pytest:
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: Test with pytest
	run: \|
	cd ${{ env.folder }}

	# Run tests
	make pip_test
	make pytest_coverage

	- name: Upload coverage reports to Codecov
	# Coverage should also be uploaded if tests still fail.
	# This will pick up the file coverage.xml from the test run
	if: always()
	uses: codecov/codecov-action@v3
	with:
	flags: ${{ env.folder }}

	cpath-pytest-gpu:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: Run GPU tests
	run: \|
	cd ${{ env.folder }}
	make pytest_gpu

	- name: Upload coverage reports to Codecov
	# Coverage should also be uploaded if tests still fail.
	# This will pick up the file pytest_gpu_coverage.xml from the GPU tests.
	if: always()
	uses: codecov/codecov-action@v3
	with:
	flags: ${{ env.folder }}

	smoke_test_cucim_slidespandaimagenetmil:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_cucim_slidespandaimagenetmil_aml

	smoke_test_openslide_slidespandaimagenetmil:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_openslide_slidespandaimagenetmil_aml

	smoke_test_tilespandaimagenetmil:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_tilespandaimagenetmil_aml

	smoke_test_tcgacrckimagenetmil:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_tcgacrckimagenetmil_aml

	smoke_test_tcgacrcksslmil:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_tcgacrcksslmil_aml

	smoke_test_crck_simclr:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_crck_simclr_aml

	smoke_test_crck_flexible_finetuning:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_crck_flexible_finetuning_aml

	smoke_test_crck_loss_analysis:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_crck_loss_analysis_aml

	smoke_test_slides_panda_loss_analysis:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_slides_panda_loss_analysis_aml

	smoke_test_slides_panda_no_ddp_sampler:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_slides_panda_no_ddp_sampler_aml

	smoke_test_tiles_panda_no_ddp_sampler:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_tiles_panda_no_ddp_sampler_aml

	smoke_test_montage_creation:
	runs-on: ubuntu-20.04
	needs: [ cancel-azureml ]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Prepare Conda environment
	uses: ./.github/actions/prepare_cpath_environment

	- name: smoke test
	run: \|
	cd ${{ env.folder }}
	make smoke_test_montage_creation

	cpath-publish-pypi-package:
	runs-on: ubuntu-20.04
	needs: [
	cpath-flake8,
	cpath-mypy,
	cpath-pytest,
	cpath-pytest-gpu,
	smoke_test_cucim_slidespandaimagenetmil,
	smoke_test_openslide_slidespandaimagenetmil,
	smoke_test_tilespandaimagenetmil,
	smoke_test_tcgacrckimagenetmil,
	smoke_test_tcgacrcksslmil,
	smoke_test_crck_simclr,
	smoke_test_crck_flexible_finetuning,
	smoke_test_crck_loss_analysis,
	smoke_test_slides_panda_loss_analysis,
	smoke_test_slides_panda_no_ddp_sampler,
	smoke_test_tiles_panda_no_ddp_sampler,
	smoke_test_montage_creation,
	]
	steps:
	- uses: actions/checkout@v3
	with:
	lfs: true

	- name: Set up Python ${{ env.pythonVersion }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.pythonVersion }}

	- name: Install build tools
	run: make pip_build

	- name: Build a binary wheel and a source tarball
	run: \|
	cd ${{ env.folder }}
	make build

	- name: Test the binary wheel
	run: \|
	cd ${{ env.folder }}
	make test_wheel

	- name: Publish package to TestPyPI
	uses: pypa/gh-action-pypi-publish@v1.8.5
	with:
	user: __token__
	password: ${{ secrets.PYPI_TEST_API_TOKEN }}
	packages_dir: ${{ env.folder }}/dist/
	repository_url: https://test.pypi.org/legacy/
	verbose: true
	skip_existing: true

	- name: Publish package to PyPI
	if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
	uses: pypa/gh-action-pypi-publish@v1.8.5
	with:
	user: __token__
	password: ${{ secrets.PYPI_API_TOKEN }}
	packages_dir: ${{ env.folder }}/dist/
	verbose: true

	cpath-pr-completed:
	runs-on: ubuntu-20.04
	if: "!startsWith(github.ref, 'refs/tags/v')"
	needs: [ cpath-publish-pypi-package ]
	steps:
	- name: Final PR Job
	run: \|
	echo "All cpath PR jobs have passed."

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ENH: Enable output datasets to store checkpoints #1778

Workflow file

ENH: Enable output datasets to store checkpoints #1778

Jobs

Run details

Workflow file for this run