Skip to content
This repository has been archived by the owner on Mar 21, 2024. It is now read-only.

Commit

Permalink
ENH: Add Environment Locking (#735)
Browse files Browse the repository at this point in the history 
* ✨ Add environment locking script

* 📝 ✨ Finish script, add documentation

* 🐛 Change Windows env file in workflow

* 📝 🐛 Add review changes + fixes

* 🚧 Temporarily alter tests and conda channels

* 🚧 Add logging

* ✅ Fix TestSubmodule env file

* 🔥 Delete env test

* 🧑‍💻 Add warning to environment.yml

* 📝 ⚰️ Update based on review comments

* 📝 Add final changes
  • Loading branch information
@peterhessey
peterhessey authored Jun 1, 2022
1 parent c7e4a3e commit 92d9479
Show file tree
Hide file tree
Showing 11 changed files with 1,138 additions and 152 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/linting_and_hello_world.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
with:
activate-environment: InnerEye
auto-activate-base: false
environment-file: environment.yml
environment-file: environment_win.yml
python-version: 3.7
if: always()

Expand Down
1 change: 1 addition & 0 deletions InnerEye/Azure/azure_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ def get_all_environment_files(project_root: Path) -> List[Path]:
files = [innereye_yaml]
if innereye_yaml != project_yaml:
files.append(project_yaml)

return files


Expand Down
1 change: 1 addition & 0 deletions InnerEye/ML/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ def after_submission_hook(azure_run: Run) -> None:
# be necessary if the innereye package is installed. It is necessary when working with an outer project
# and InnerEye as a git submodule and submitting jobs from the local machine.
# In case of version conflicts, the package version in the outer project is given priority.
logging.info(f"Attempting to merge the following conda files: {source_config.conda_dependencies_files}")
merge_conda_files(source_config.conda_dependencies_files, temp_conda)

# Calls like `self.azure_config.get_workspace()` will fail if we have no AzureML credentials set up, and so
Expand Down
310 changes: 307 additions & 3 deletions TestSubmodule/environment.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,311 @@
name: TestSubmodule
# WARNING - DO NOT EDIT THIS FILE MANUALLY
# Please refer to the environment documentation for instructions on how to create a new version of this file: https://github.com/microsoft/InnerEye-DeepLearning/blob/main/docs/environment.md
name: InnerEye
channels:
- defaults
- pytorch
- defaults
dependencies:
- pip=20.1.1
- _libgcc_mutex=0.1=main
- _openmp_mutex=5.1=1_gnu
- blas=1.0=mkl
- blosc=1.21.0=h4ff587b_1
- bzip2=1.0.8=h7b6447c_0
- ca-certificates=2022.4.26=h06a4308_0
- certifi=2022.5.18.1=py37h06a4308_0
- cudatoolkit=11.3.1=h2bc3f7f_2
- ffmpeg=4.2.2=h20bf706_0
- freetype=2.11.0=h70c0345_0
- giflib=5.2.1=h7b6447c_0
- gmp=6.2.1=h2531618_2
- gnutls=3.6.15=he1e5248_0
- intel-openmp=2021.4.0=h06a4308_3561
- jpeg=9e=h7f8727e_0
- lame=3.100=h7b6447c_0
- lcms2=2.12=h3be6417_0
- libedit=3.1.20210910=h7f8727e_0
- libffi=3.2.1=hf484d3e_1007
- libgcc-ng=11.2.0=h1234567_0
- libgomp=11.2.0=h1234567_0
- libidn2=2.3.2=h7f8727e_0
- libopus=1.3.1=h7b6447c_0
- libpng=1.6.37=hbc83047_0
- libstdcxx-ng=11.2.0=h1234567_0
- libtasn1=4.16.0=h27cfd23_0
- libtiff=4.2.0=h2818925_1
- libunistring=0.9.10=h27cfd23_0
- libuv=1.40.0=h7b6447c_0
- libvpx=1.7.0=h439df22_0
- libwebp=1.2.2=h55f646e_0
- libwebp-base=1.2.2=h7f8727e_0
- lz4-c=1.9.3=h295c915_1
- mkl=2021.4.0=h06a4308_640
- mkl-service=2.4.0=py37h7f8727e_0
- mkl_fft=1.3.1=py37hd3c417c_0
- mkl_random=1.2.2=py37h51133e4_0
- ncurses=6.3=h7f8727e_2
- nettle=3.7.3=hbbd107a_1
- openh264=2.1.1=h4ff587b_0
- openssl=1.1.1o=h7f8727e_0
- pip=20.1.1=py37_1
- python=3.7.3
- python-blosc=1.7.0=py37h7b6447c_0
- pytorch=1.10.0=py3.7_cuda11.3_cudnn8.2.0_0
- pytorch-mutex=1.0=cuda
- readline=7.0=h7b6447c_5
- setuptools=61.2.0=py37h06a4308_0
- sqlite=3.33.0=h62c20be_0
- tk=8.6.11=h1ccaba5_1
- torchvision=0.11.1=py37_cu113
- typing_extensions=4.1.1=pyh06a4308_0
- wheel=0.37.1=pyhd3eb1b0_0
- x264=1!157.20191217=h7b6447c_0
- xz=5.2.5=h7f8727e_1
- zlib=1.2.12=h7f8727e_2
- zstd=1.5.2=ha4553b6_0
- pip:
- absl-py==1.0.0
- adal==1.2.7
- aiohttp==3.8.1
- aiosignal==1.2.0
- alembic==1.7.7
- ansiwrap==0.8.4
- applicationinsights==0.11.10
- argon2-cffi==21.3.0
- argon2-cffi-bindings==21.2.0
- async-timeout==4.0.2
- asynctest==0.13.0
- attrs==21.4.0
- azure-common==1.1.28
- azure-core==1.24.0
- azure-graphrbac==0.61.1
- azure-identity==1.7.0
- azure-mgmt-authorization==0.61.0
- azure-mgmt-containerregistry==10.0.0
- azure-mgmt-core==1.3.0
- azure-mgmt-datafactory==1.1.0
- azure-mgmt-keyvault==9.3.0
- azure-mgmt-resource==12.1.0
- azure-mgmt-storage==11.2.0
- azure-storage-blob==12.6.0
- azureml-automl-core==1.36.1
- azureml-core==1.36.0.post2
- azureml-dataprep==2.24.4
- azureml-dataprep-native==38.0.0
- azureml-dataprep-rslex==2.0.3
- azureml-dataset-runtime==1.36.0
- azureml-mlflow==1.36.0
- azureml-pipeline==1.36.0
- azureml-pipeline-core==1.36.0
- azureml-pipeline-steps==1.36.0
- azureml-sdk==1.36.0
- azureml-telemetry==1.36.0
- azureml-tensorboard==1.36.0
- azureml-train-automl-client==1.36.0
- azureml-train-core==1.36.0
- azureml-train-restclients-hyperdrive==1.36.0
- backcall==0.2.0
- backports-tempfile==1.0
- backports-weakref==1.0.post1
- beautifulsoup4==4.11.1
- black==22.3.0
- bleach==5.0.0
- cachetools==4.2.4
- cffi==1.15.0
- charset-normalizer==2.0.12
- click==8.1.3
- cloudpickle==1.6.0
- colorama==0.4.4
- commonmark==0.9.1
- conda-merge==0.1.5
- contextlib2==21.6.0
- coverage==6.4
- cryptography==3.3.2
- cucim==21.10.1
- cycler==0.11.0
- databricks-cli==0.16.6
- dataclasses-json==0.5.2
- debugpy==1.6.0
- decorator==5.1.1
- defusedxml==0.7.1
- deprecated==1.2.13
- distro==1.7.0
- docker==4.3.1
- dotnetcore2==2.1.23
- entrypoints==0.4
- execnet==1.9.0
- fastjsonschema==2.15.3
- flake8==3.8.3
- flask==2.1.2
- frozenlist==1.3.0
- fsspec==2022.5.0
- fusepy==3.0.1
- future==0.18.2
- gitdb==4.0.9
- gitpython==3.1.7
- google-auth==1.35.0
- google-auth-oauthlib==0.4.6
- gputil==1.4.0
- greenlet==1.1.2
- grpcio==1.46.3
- gunicorn==20.1.0
- h5py==2.10.0
- humanize==4.1.0
- idna==3.3
- imageio==2.15.0
- importlib-metadata==4.11.4
- importlib-resources==5.7.1
- iniconfig==1.1.1
- innereye-dicom-rt==1.0.3
- ipykernel==6.13.0
- ipython==7.31.1
- ipython-genutils==0.2.0
- ipywidgets==7.7.0
- isodate==0.6.1
- itsdangerous==2.1.2
- jedi==0.18.1
- jeepney==0.8.0
- jinja2==3.1.2
- jmespath==0.10.0
- joblib==0.16.0
- jsonpickle==2.2.0
- jsonschema==4.5.1
- jupyter==1.0.0
- jupyter-client==6.1.5
- jupyter-console==6.4.3
- jupyter-core==4.10.0
- jupyterlab-pygments==0.2.2
- jupyterlab-widgets==1.1.0
- kiwisolver==1.4.2
- lightning-bolts==0.4.0
- llvmlite==0.34.0
- mako==1.2.0
- markdown==3.3.7
- markupsafe==2.1.1
- marshmallow==3.16.0
- marshmallow-enum==1.5.1
- matplotlib==3.3.0
- matplotlib-inline==0.1.3
- mccabe==0.6.1
- mistune==0.8.4
- mlflow==1.23.1
- mlflow-skinny==1.26.1
- monai==0.6.0
- more-itertools==8.13.0
- msal==1.17.0
- msal-extensions==0.3.1
- msrest==0.6.21
- msrestazure==0.6.4
- multidict==6.0.2
- mypy==0.910
- mypy-extensions==0.4.3
- nbclient==0.6.3
- nbconvert==6.5.0
- nbformat==5.4.0
- ndg-httpsclient==0.5.1
- nest-asyncio==1.5.5
- networkx==2.6.3
- nibabel==3.2.2
- notebook==6.4.11
- numba==0.51.2
- numpy==1.19.1
- oauthlib==3.2.0
- opencv-python-headless==4.5.1.48
- packaging==21.3
- pandas==1.1.0
- pandocfilters==1.5.0
- papermill==2.2.2
- param==1.9.3
- parso==0.8.3
- pathspec==0.9.0
- pexpect==4.8.0
- pickleshare==0.7.5
- pillow==9.0.0
- platformdirs==2.5.2
- pluggy==0.13.1
- portalocker==2.4.0
- prometheus-client==0.14.1
- prometheus-flask-exporter==0.20.1
- prompt-toolkit==3.0.29
- protobuf==3.20.1
- psutil==5.7.2
- ptyprocess==0.7.0
- py==1.11.0
- pyarrow==3.0.0
- pyasn1==0.4.8
- pyasn1-modules==0.2.8
- pycodestyle==2.6.0
- pycparser==2.21
- pydeprecate==0.3.1
- pydicom==2.0.0
- pyflakes==2.2.0
- pygments==2.12.0
- pyjwt==1.7.1
- pynndescent==0.5.7
- pyopenssl==20.0.1
- pyparsing==3.0.9
- pyrsistent==0.18.1
- pytest==6.0.1
- pytest-cov==2.10.1
- pytest-forked==1.3.0
- pytest-xdist==1.34.0
- python-dateutil==2.8.2
- pytorch-lightning==1.5.5
- pytz==2022.1
- pywavelets==1.3.0
- pyyaml==6.0
- pyzmq==23.0.0
- qtconsole==5.3.0
- qtpy==2.1.0
- querystring-parser==1.2.4
- requests==2.27.1
- requests-oauthlib==1.3.1
- rich==10.13.0
- rpdb==0.1.6
- rsa==4.8
- ruamel-yaml==0.16.12
- ruamel-yaml-clib==0.2.6
- runstats==1.8.0
- scikit-image==0.17.2
- scikit-learn==0.23.2
- scipy==1.5.2
- seaborn==0.10.1
- secretstorage==3.3.2
- send2trash==1.8.0
- simpleitk==1.2.4
- six==1.15.0
- smmap==5.0.0
- soupsieve==2.3.2.post1
- sqlalchemy==1.4.36
- sqlparse==0.4.2
- stopit==1.1.2
- stringcase==1.2.0
- tabulate==0.8.7
- tenacity==8.0.1
- tensorboard==2.3.0
- tensorboard-plugin-wit==1.8.1
- tensorboardx==2.1
- terminado==0.15.0
- textwrap3==0.9.2
- threadpoolctl==3.1.0
- tifffile==2021.11.2
- tinycss2==1.1.1
- toml==0.10.2
- tomli==2.0.1
- torchio==0.18.74
- torchmetrics==0.6.0
- tornado==6.1
- tqdm==4.64.0
- traitlets==5.2.1.post0
- typed-ast==1.4.3
- typing-inspect==0.7.1
- umap-learn==0.5.2
- urllib3==1.26.7
- wcwidth==0.2.5
- webencodings==0.5.1
- websocket-client==1.3.2
- werkzeug==2.1.2
- widgetsnbextension==3.6.0
- wrapt==1.14.1
- yacs==0.1.8
- yarl==1.7.2
- zipp==3.8.0
41 changes: 0 additions & 41 deletions Tests/Common/test_environment.py
View file

This file was deleted.

Loading

0 comments on commit 92d9479

Please sign in to comment.