Package Sync (#235)

coiled · Sep 15, 2022 · f9f8d9b · f9f8d9b
1 parent 57f01ac
commit f9f8d9b
Show file tree

Hide file tree

Showing 13 changed files with 133 additions and 261 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -4,7 +4,7 @@ on:
     branches:
       - main
     tags:
-      - '*'
+      - "*"
   pull_request:
   schedule:
     # Runs "At 00:01" (see https://crontab.guru)
@@ -22,104 +22,16 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  software:
-    name: Setup
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set up environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniforge-variant: Mambaforge
-          use-mamba: true
-          condarc-file: ci/condarc
-          python-version: ${{ matrix.python-version }}
-          environment-file: ci/environment.yml
-
-      - name: Check upstream
-        run: |
-          if [[ ${{ github.event_name }} == "pull_request" ]]
-          then
-              REF="HEAD^2"
-          else
-              REF="HEAD"
-          fi
-
-          COMMIT="$(git log -n 1 --pretty=format:%s $REF)"
-          if [[ "$COMMIT" == *"test-upstream"* || ${{ github.event_name }} == "schedule" ]]
-          then
-            TEST_UPSTREAM="true"
-            AB_BASELINE="coiled-upstream-py3.9 coiled-0.1.0-py3.9"
-          else
-            TEST_UPSTREAM="false"
-            AB_BASELINE="coiled-latest-py3.9 coiled-0.1.0-py3.9"
-          fi
-
-          # Put TEST_UPSTREAM into $GITHUB_ENV so it can be used in subsequent workflow steps
-          echo $TEST_UPSTREAM
-          echo TEST_UPSTREAM=$TEST_UPSTREAM >> $GITHUB_ENV
-
-          # Put env variables into files so it can be downloaded in subsequent workflow jobs
-          echo $TEST_UPSTREAM > test_upstream.txt
-          echo $AB_BASELINE > ab_baseline.txt
-
-      - name: Build Coiled Software Environment
-        env:
-          DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
-        run: |
-          PYTHON_VERSION_FORMATTED=$(echo "${{ matrix.python-version }}" | sed 's/\.//g' )
-          NAME_HEAD=dask-engineering/coiled-runtime-${{ github.event_name }}
-          NAME_TAIL=$GITHUB_RUN_ID-py$PYTHON_VERSION_FORMATTED
-          if [[ ${{ github.event_name }} = 'pull_request' ]]
-          then
-            NAME_MID=${{ github.event.number }}
-          else
-            NAME_MID=$GITHUB_REF_TYPE-$(echo "$GITHUB_REF_NAME" | sed 's/\./-/g' )
-          fi
-          # env name can only contain lowercase ASCII letters, numbers, hyphens and underscores
-          COILED_SOFTWARE_NAME=$NAME_HEAD-$(echo $NAME_MID-$NAME_TAIL | tr 'A-Z' 'a-z' | sed -r 's/[^a-z0-9_-]/_/g')
-
-          # Create conda environment.yaml file for the latest software environment
-          python ci/create_latest_runtime_meta.py
-          ENV_FILE=coiled_software_environment.yaml
-          cat $ENV_FILE
-
-          mamba install coiled
-          echo "Creating Coiled software environment for $COILED_SOFTWARE_NAME"
-          coiled env create --name $COILED_SOFTWARE_NAME --conda $ENV_FILE
-
-          # Put COILED_SOFTWARE_NAME into a file so it can be downloaded in subsequent workflow jobs
-          echo $COILED_SOFTWARE_NAME > software_name.txt
-
-      - name: Upload environment file
-        uses: actions/upload-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-          path: |
-            coiled_software_environment.yaml
-            software_name.txt
-            test_upstream.txt
-            ab_baseline.txt
-
   runtime:
     name: Runtime - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.9"]
-        runtime-version: ["latest", "0.0.4", "0.1.0"]
+        runtime-version: ["upstream", "latest", "0.0.4", "0.1.0"]
 
     steps:
       - uses: actions/checkout@v2
@@ -135,16 +47,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run Coiled Runtime Tests
         id: test
@@ -166,15 +74,14 @@ jobs:
 
   benchmarks:
     name: Benchmarks - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.9"]
-        runtime-version: ["latest", "0.0.4", "0.1.0"]
+        runtime-version: ["upstream", "latest", "0.0.4", "0.1.0"]
 
     steps:
       - uses: actions/checkout@v2
@@ -190,16 +97,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run benchmarking tests
         id: benchmarking_tests
@@ -221,15 +124,14 @@ jobs:
 
   stability:
     name: Stability - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
-    needs: software
     runs-on: ${{ matrix.os }}
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.8", "3.9", "3.10"]
-        runtime-version: ["latest", "0.0.4", "0.1.0"]
+        runtime-version: ["upstream", "latest", "0.0.4", "0.1.0"]
         include:
           - python-version: "3.9"
             runtime-version: "latest"
@@ -252,16 +154,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: ci/environment.yml
 
-      - name: Download software environment assets
-        if: matrix.runtime-version == 'latest'
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
       - name: Install coiled-runtime
         env:
           COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
-        run: source ci/scripts/install_coiled_runtime.sh
+        run: |
+          python ci/create_runtime_meta.py
+          source ci/scripts/install_coiled_runtime.sh
 
       - name: Run stability tests
         id: stability_tests
@@ -282,39 +180,6 @@ jobs:
           name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
           path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
 
-  cleanup:
-    needs: [software, runtime, benchmarks, stability]
-    if: always()
-    name: Cleanup
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install coiled
-        run: python -m pip install coiled
-
-      - name: Download software environment assets
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py${{ matrix.python-version }}
-
-      - name: Remove Coiled software environment
-        env:
-          DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
-        run: |
-          SOFTWARE_NAME=$(cat software_name.txt)
-          echo "Deleting $SOFTWARE_NAME"
-          coiled env delete $SOFTWARE_NAME
-
   process-results:
     needs: [runtime, benchmarks, stability]
     name: Combine separate benchmark results
@@ -341,7 +206,7 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-2  # this is needed for boto for some reason
+          AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           DB_NAME: benchmark.db
         run: |
           aws s3 cp s3://coiled-runtime-ci/benchmarks/$DB_NAME . || true
@@ -356,15 +221,15 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-2  # this is needed for boto for some reason
+          AWS_DEFAULT_REGION: us-east-2 # this is needed for boto for some reason
           DB_NAME: benchmark.db
         run: |
           aws s3 cp $DB_NAME s3://coiled-runtime-ci/benchmarks/
 
       - name: Upload benchmark results as artifact
         uses: actions/upload-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
           path: benchmark.db
 
   regressions:
@@ -380,7 +245,7 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
 
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v2
@@ -451,7 +316,7 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: benchmark.db
+          name: benchmark
 
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v2
@@ -461,13 +326,8 @@ jobs:
           python-version: "3.9"
           environment-file: ci/environment-dashboard.yml
 
-      - name: Download software environment assets
-        uses: actions/download-artifact@v3
-        with:
-          name: software-environment-py3.9
-
       - name: Generate dashboards
-        run: python dashboard.py -d benchmark.db -o static -b $(cat ab_baseline.txt)
+        run: python dashboard.py -d benchmark.db -o static -b coiled-latest-py3.9 coiled-upstream-py3.9 
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3

diff --git a/README.md b/README.md
@@ -57,10 +57,11 @@ The `coiled-runtime` test suite can be run locally with the following steps:
    the Coiled Dask Engineering AWS S3 account.
 2. Create a Python environment and install development dependencies as
    specified in `ci/environment.yml`.
-3. (Optional) If testing against an unreleased version of `coiled-runtime`,
-   create a Coiled software environment with the unreleased `coiled-runtime` installed
-   and set a local `COILED_SOFTWARE_NAME` environment variable to the name
-   of the software environment (e.g. `export COILED_SOFTWARE_NAME="account/software-name"`)
+3. Install a coiled runtime environment. This might be from one of the environments
+   listed in ``environments/``, or it could be a development environment if you are
+   testing feature branches of dask or distributed. This test suite is configured
+   to run Coiled's ``package_sync`` feature, so your local environment should be copied
+   to the cluster.
 4. Run tests with `python -m pytest tests`
 
 Additionally, tests are automatically run on pull requests to this repository.

diff --git a/alembic/versions/2764a4f5582b_declare_bankruptcy_for_cluster_startup_.py b/alembic/versions/2764a4f5582b_declare_bankruptcy_for_cluster_startup_.py
@@ -0,0 +1,30 @@
+"""Declare bankruptcy for cluster startup time
+
+Revision ID: 2764a4f5582b
+Revises: 924e9b1430e1
+Create Date: 2022-09-14 11:45:46.024184
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "2764a4f5582b"
+down_revision = "924e9b1430e1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        delete from test_run
+        where originalname = 'test_default_cluster_spinup_time'
+        and path = 'benchmarks/test_coiled.py';
+        """
+    )
+
+
+def downgrade() -> None:
+    pass
diff --git a/ci/create_latest_runtime_meta.py → ci/create_runtime_meta.py b/ci/create_latest_runtime_meta.py → ci/create_runtime_meta.py
@@ -6,7 +6,6 @@
 import shlex
 import subprocess
 import sys
-from distutils.util import strtobool
 
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -38,15 +37,20 @@ def main():
         if package_name == "python":
             requirements[idx] = f"python =={python_version}"
 
-    # Optionally use the development version of `dask` and `distributed`
-    # from `dask/label/dev` conda channel
-    upstream = strtobool(os.environ.get("TEST_UPSTREAM", "false"))
-    if upstream:
-        upstream_packages = {"dask", "distributed"}
-        for idx, req in enumerate(requirements):
-            package_name = Requirement(req).name
-            if package_name in upstream_packages:
-                requirements[idx] = get_latest_conda_build(package_name)
+    if os.environ.get("COILED_RUNTIME_VERSION", "unknown") == "upstream":
+        requirements = [
+            r
+            for r in requirements
+            if Requirement(r).name not in {"dask", "distributed"}
+        ]
+        requirements.append(
+            {
+                "pip": [
+                    "git+https://github.com/dask/dask@main",
+                    "git+https://github.com/dask/distributed@main",
+                ]
+            }
+        )
 
     # File compatible with `mamba env create --file <...>`
     env = {