From 8bc64afaf1812446b122a37be41bd1661c199404 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 11:43:41 +0200 Subject: [PATCH 01/20] create .github/workflows/run_workflow.yml --- .github/workflows/run_workflow.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/workflows/run_workflow.yml diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml new file mode 100644 index 0000000..e69de29 From cfd80af03c8a0924a36f7227ef7388a4132d3351 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 11:54:51 +0200 Subject: [PATCH 02/20] add environment.yml --- .github/workflows/run_workflow.yml | 0 environment.yml | 4 ++++ 2 files changed, 4 insertions(+) delete mode 100644 .github/workflows/run_workflow.yml create mode 100644 environment.yml diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml deleted file mode 100644 index e69de29..0000000 diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..1f735a7 --- /dev/null +++ b/environment.yml @@ -0,0 +1,4 @@ +channels: + - conda-forge +dependencies: + - pyiron_workflow From d5d1cd35b0d5e180feed623e5df5e442a674b723 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 11:55:46 +0200 Subject: [PATCH 03/20] create .github/workflows/run_workflow.yml --- .github/workflows/run_workflow.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/run_workflow.yml diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml new file mode 100644 index 0000000..43d9b44 --- /dev/null +++ b/.github/workflows/run_workflow.yml @@ -0,0 +1,26 @@ +name: Run workflow + +on: + push: + branches: [ main ] + pull_request: + types: [labeled, opened, synchronize, reopened] + +jobs: + build: + if: | + github.event_name == 'push' || + ( github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run_workflow' )) + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2.2.0 + with: + python-version: "3.11" + mamba-version: "*" + miniforge-variant: Mambaforge + channels: conda-forge + channel-priority: strict + auto-update-conda: true + environment-file: environment.yml From a49e1a9ef1245c3e60d47b7b9558f8b4b6810d74 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 11:57:45 +0200 Subject: [PATCH 04/20] create gitignore (just so that it's there) --- .gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 From 8f08a8cb596d0af5fd1bdc4e223a6889ddd3b201 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:09:17 +0200 Subject: [PATCH 05/20] let the workflow be executed when the label run_workflow is added to a PR --- .github/workflows/run_workflow.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 43d9b44..e4cb7bb 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -5,22 +5,17 @@ on: branches: [ main ] pull_request: types: [labeled, opened, synchronize, reopened] + workflow_call: jobs: build: if: | github.event_name == 'push' || ( github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run_workflow' )) - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2.2.0 - with: - python-version: "3.11" - mamba-version: "*" - miniforge-variant: Mambaforge - channels: conda-forge - channel-priority: strict - auto-update-conda: true - environment-file: environment.yml + - uses: actions/checkout@v4 + - uses: pyiron/actions/build-notebooks@actions-3.1.0 + with: + python-version: '3.11' + env-files: environment.yml From 0906b630d2da0fae122e6322162b1c82560954e8 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:16:02 +0200 Subject: [PATCH 06/20] adjust path to env file --- .github/workflows/run_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index e4cb7bb..208479d 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -18,4 +18,4 @@ jobs: - uses: pyiron/actions/build-notebooks@actions-3.1.0 with: python-version: '3.11' - env-files: environment.yml + env-files: ./environment.yml From 3dac0da861b360746c140deb6695f1a69330b6e3 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:22:09 +0200 Subject: [PATCH 07/20] ??? --- .ci_support/environment.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .ci_support/environment.yml diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml new file mode 100644 index 0000000..1f735a7 --- /dev/null +++ b/.ci_support/environment.yml @@ -0,0 +1,4 @@ +channels: + - conda-forge +dependencies: + - pyiron_workflow From f1e6233637864d667cb164d0d16a1fb85b24294f Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:22:58 +0200 Subject: [PATCH 08/20] forgot to change this in the gh-workflow file --- .github/workflows/run_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 208479d..c1cc662 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -18,4 +18,4 @@ jobs: - uses: pyiron/actions/build-notebooks@actions-3.1.0 with: python-version: '3.11' - env-files: ./environment.yml + env-files: .ci_support/environment.yml From 0b9ea6ea01b7e4d435d08cc4cf6dd067ff5bd0cc Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:33:22 +0200 Subject: [PATCH 09/20] update pyiron/actions release --- .github/workflows/run_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index c1cc662..97c6ddb 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pyiron/actions/build-notebooks@actions-3.1.0 + - uses: pyiron/actions/build-notebooks@actions-3.2.0 with: python-version: '3.11' - env-files: .ci_support/environment.yml + env-files: environment.yml From fa7cfc08eb8484392cdd18e18dc10cfbdfdd98f2 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:37:04 +0200 Subject: [PATCH 10/20] remove environment.yml form root dir; located now in .ci_support/ --- environment.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 environment.yml diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 1f735a7..0000000 --- a/environment.yml +++ /dev/null @@ -1,4 +0,0 @@ -channels: - - conda-forge -dependencies: - - pyiron_workflow From efd2d54f9b2808ca3da8c3297285c5c874772257 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:37:41 +0200 Subject: [PATCH 11/20] account for this in the action --- .github/workflows/run_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 97c6ddb..2cfbc19 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -18,4 +18,4 @@ jobs: - uses: pyiron/actions/build-notebooks@actions-3.2.0 with: python-version: '3.11' - env-files: environment.yml + env-files: .ci_support/environment.yml From f0e50924df8326bd1033e6731c284bc1402b29d1 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 12:57:20 +0200 Subject: [PATCH 12/20] use another action --- .github/workflows/run_workflow.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 2cfbc19..e32c922 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -8,14 +8,22 @@ on: workflow_call: jobs: - build: + run_workflow: if: | github.event_name == 'push' || ( github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run_workflow' )) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pyiron/actions/build-notebooks@actions-3.2.0 + - name: Setup environment + uses: pyiron/actions/cached-miniforge@actions-3.2.0 with: python-version: '3.11' env-files: .ci_support/environment.yml + use-cache: 'true' + miniforge-channels: conda-forge + pip-install-versioneer: 'false' + no-build-isolation: 'false' + - name: Execute notebook + shell: bash -l {0} + run: jupyter nbconvert --execute --to notebook workflow.ipynb From b023189cb4602c65c3fd82d8a1a7763fabf4ffae Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 13:04:48 +0200 Subject: [PATCH 13/20] no local code to be pip-installed --- .github/workflows/run_workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index e32c922..37b9af1 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -20,6 +20,7 @@ jobs: with: python-version: '3.11' env-files: .ci_support/environment.yml + local-code-directory: '' use-cache: 'true' miniforge-channels: conda-forge pip-install-versioneer: 'false' From 1cc4d8f4f788651ea06eff9700910e7283a2c97a Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 13:28:56 +0200 Subject: [PATCH 14/20] install jupyter from within the gh-workflow def --- .github/workflows/run_workflow.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 37b9af1..fa32d1b 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -6,7 +6,7 @@ on: pull_request: types: [labeled, opened, synchronize, reopened] workflow_call: - + jobs: run_workflow: if: | @@ -25,6 +25,9 @@ jobs: miniforge-channels: conda-forge pip-install-versioneer: 'false' no-build-isolation: 'false' - - name: Execute notebook + - name: Add jupyter + shell: bash -l {0} + run: mamba install -n my-env jupyter + - name: Execute workflow shell: bash -l {0} run: jupyter nbconvert --execute --to notebook workflow.ipynb From 26e00d909a3db4b8901b4fd27f521ac1aacae056 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 13:36:48 +0200 Subject: [PATCH 15/20] explicitly specify the kernel to be used --- .github/workflows/run_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index fa32d1b..891d115 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -30,4 +30,4 @@ jobs: run: mamba install -n my-env jupyter - name: Execute workflow shell: bash -l {0} - run: jupyter nbconvert --execute --to notebook workflow.ipynb + run: jupyter nbconvert --ExecutePreprocessor.kernel_name=python3 --execute --to notebook workflow.ipynb From 9e251661e898b9747ff30fac1e987ea85eb180c4 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 13:45:17 +0200 Subject: [PATCH 16/20] testing: make a trivial change to a node --- nodes/time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/time.py b/nodes/time.py index 8aa303f..587ad11 100644 --- a/nodes/time.py +++ b/nodes/time.py @@ -2,4 +2,4 @@ @Workflow.wrap.as_function_node("travel_time") def get_time(distance: float, speed: float) -> float: - return 10. + return 42. From f7a804a7d0227da2f6e4b2871a3943d43072c9b4 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 13:59:24 +0200 Subject: [PATCH 17/20] papermill instead of jupyter nbconvert --- .github/workflows/run_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 891d115..939b5ee 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -27,7 +27,7 @@ jobs: no-build-isolation: 'false' - name: Add jupyter shell: bash -l {0} - run: mamba install -n my-env jupyter + run: mamba install -n my-env papermill - name: Execute workflow shell: bash -l {0} - run: jupyter nbconvert --ExecutePreprocessor.kernel_name=python3 --execute --to notebook workflow.ipynb + run: papermill workflow.ipynb workflow-out.ipynb From 49f6aeb5b0a4a2b21858ab6bae6ce0eaad66b853 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 14:01:40 +0200 Subject: [PATCH 18/20] rename corresponding section in the action def; specify kernel name --- .github/workflows/run_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 939b5ee..75f793b 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -25,9 +25,9 @@ jobs: miniforge-channels: conda-forge pip-install-versioneer: 'false' no-build-isolation: 'false' - - name: Add jupyter + - name: Add papermill shell: bash -l {0} run: mamba install -n my-env papermill - name: Execute workflow shell: bash -l {0} - run: papermill workflow.ipynb workflow-out.ipynb + run: papermill workflow.ipynb workflow-out.ipynb -k python3 From 69a74f9fee709f483df8696146e03fcfdf25fa18 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 14:11:51 +0200 Subject: [PATCH 19/20] upload the updated notebook as artifact --- .github/workflows/run_workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run_workflow.yml b/.github/workflows/run_workflow.yml index 75f793b..1473137 100644 --- a/.github/workflows/run_workflow.yml +++ b/.github/workflows/run_workflow.yml @@ -31,3 +31,8 @@ jobs: - name: Execute workflow shell: bash -l {0} run: papermill workflow.ipynb workflow-out.ipynb -k python3 + - name: Upload updated notebook as artifact + uses: actions/upload-artifact@v4 + with: + name: workflow-out.ipynb + path: ./workflow-out.ipynb From eb6b6602b552974c1a9ca15d8c559d2f0d5bde55 Mon Sep 17 00:00:00 2001 From: mbruns91 Date: Mon, 17 Jun 2024 14:58:50 +0200 Subject: [PATCH 20/20] update README.md --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2b91c3c..1399409 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # graphathon -Playground for collaborative environment in order to create a workflow in parallel by distributing nodes (i.e. tasks) to different people. - -## Basic idea - -1. Create a workflow in a large group, with empty nodes with only inputs and outputs defined -2. Distribute the nodes to different people -3. As people finish their tasks, they push the nodes to the repository, where CI takes care of the workflow execution - -More will follow... +A template repository for setting up an environment where nodes for a workflow using `pyiron_workflow` can be developed by different people in parallel. + +Usage goes like so: +- the team sets up a dummy workflow using placeholder-vwersions of the involved nodes +- the team creates a repository using *this* repo as a template (especially anything located in `.github/` and `.ci_support/`) +- each dev opens a branch + PR for her work +- as soon as the `run_workflow` label is added to the PR, each push triggers workflow execution via actions. +- the updated notebook is uploaded as a github action artifact (Actions -> select CI run -> Artifacts). Download, unzip and view locally.