From e8015101a7f028fb1b0aa53ba00c30c817a63f46 Mon Sep 17 00:00:00 2001
From: Ben Chambers <35960+bjchambers@users.noreply.github.com>
Date: Thu, 1 Feb 2024 10:45:52 -0800
Subject: [PATCH] ci: Initial release workflows (#59)

* ci: Initial release workflows

These are currently configured to publish `dewy` and `dewy-client`.
---
 .github/workflows/README.md   |   1 +
 .github/workflows/ci.yml      |   2 +-
 .github/workflows/release.yml | 178 ++++++++++++++++++++++++++++++++++
 .gitignore                    |   4 +-
 README.md                     |  20 +++-
 5 files changed, 199 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index ccd41db..ecb1802 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -9,6 +9,7 @@ We currently have the following workflows:
 3. `site.yml` defines the build and deployment process for the site.
 4. `release-drafter.yml`: Generates draft release notes as PRs are merged.
 5. `sync-labels.yml`: Updates the labels in the project to match `.github/labels.yml`.
+6. `release.yml`: Release automation triggered when a release is created.
 
 ## Future Improvements
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa79cbb..1df6f86 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,7 +42,7 @@ jobs:
         run: |
           poetry run poe extract-openapi
           poetry run poe update-client
-          # Record intent to add any new files in client
+          # Record intent to add any new files in `dewy-client`
           git add -N dewy-client
           # Diff, and report any changes (including any new files in dewy-client)
           git diff --exit-code
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..f0c215e
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,178 @@
+name: Release
+
+# Only one release job at a time.
+concurrency:
+    group: ${{ github.workflow }}
+    cancel-in-progress: true
+
+# Triggered when a release is published.defaults:
+# This happens when a draft (generated by release-drafter.yml) is published.
+#
+# This applies whether the published draft is marked "pre-release" or not.
+#
+# > NOTE: The prereleased type will not trigger for pre-releases published from
+# > draft releases, but the published type will trigger. If you want a workflow
+# > to run when stable and pre-releases publish, subscribe to published instead
+# > of released and prereleased.
+# >
+# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#release
+on:
+  push:
+    branches:
+      - main
+  release:
+    types:
+    - published
+
+# Generally limited permissions.
+# We'll request write permission when needed for deployment.
+permissions:
+  contents: read
+
+# Basic workflow from
+# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#configuring-trusted-publishing
+jobs:
+  build-dewy:
+    name: Build dewy (and dewy-client) distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+      - name: Build dewy
+        run: |
+          poetry install --with=dev
+          poetry build
+      - name: Store the dewy distribution packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: dewy-distributions
+          path: dist/
+      - name: Build dewy-client
+        working-directory: dewy-client
+        run: |
+          poetry install
+          poetry build
+      - name: Store the dewy-client distribution packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: dewy-client-distributions
+          path: dewy-client/dist/
+
+  publish-dewy-to-pypi:
+    name: Publish dewy to pypi
+    # Only publish to PyPi on releases.
+    if: github.event_name == 'release' && github.event.action == 'published'
+    needs: build-dewy
+    runs-on: ubuntu-latest
+    environment:
+      # name: test-pypi
+      # url: https://test.pypi.org/p/dewy
+      name: prod-pypi
+      url: https://pypi.org/p/dewy
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download the dewy dist
+        uses: actions/download-artifact@v3
+        with:
+          name: dewy-distributions
+          path: dist/
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # with:
+          # repository-url: https://test.pypi.org/legacy/
+
+  publish-dewy-client-to-pypi:
+    name: Publish dewy-client to pypi
+    # Only publish to PyPi on releases.
+    if: github.event_name == 'release' && github.event.action == 'published'
+    needs: build-dewy
+    runs-on: ubuntu-latest
+    environment:
+      # name: test-pypi
+      # url: https://test.pypi.org/p/dewy-client
+      name: prod-pypi
+      url: https://pypi.org/p/dewy-client
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download the dewy-client dist
+        uses: actions/download-artifact@v3
+        with:
+          name: dewy-client-distributions
+          path: dewy-client/dist/
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dewy-client/dist
+          # repository-url: https://test.pypi.org/legacy/
+
+  github-release:
+    name: Sign and finalize GitHub release
+    # Only publish on release.
+    if: github.event_name == 'release' && github.event.action == 'published'
+    needs:
+    - publish-dewy-to-pypi
+    - publish-dewy-client-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download the dewy dist
+      uses: actions/download-artifact@v3
+      with:
+        name: dewy-distributions
+        path: dist/
+    - name: Download the dewy-client dist
+      uses: actions/download-artifact@v3
+      with:
+        name: dewy-client-distributions
+        path: dewy-client/dist/
+    - name: Sign the dewy and dewy-client dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v1.2.3
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+          ./dewy-client/dist/*.tar.gz
+          ./dewy-client/dist/*.whl
+    - name: Upload dewy artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'
+
+    - name: Upload dewy-client artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dewy-client/dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dewy-client/dist/**
+        --repo '${{ github.repository }}'
+
+    - name: Publish release
+      # TODO: Add --discussion-category "Announcements" to create a release discussion?
+      run: |
+        gh release edit ${{ github.ref_name }} \
+          --draft=false --latest
+      env:
+        GH_TOKEN: ${{ github.token }}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 6150e7b..1a61a3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 .ruff_cache
 __pycache__
 .env
-.vscode/
\ No newline at end of file
+.vscode/
+/dist
+/dewy-client/dist
\ No newline at end of file
diff --git a/README.md b/README.md
index 7c31197..1417445 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ To get a local copy up and running follow these steps.
 
 1. (Optional) Start a `pgvector` instance to persist your data
 
-    Dewy uses a vector database to store metadata about the documents you've loaded as well as embeddings used to provide semantic search results. 
+    Dewy uses a vector database to store metadata about the documents you've loaded as well as embeddings used to provide semantic search results.
 
     ```sh
     docker run -d \
@@ -61,7 +61,7 @@ To get a local copy up and running follow these steps.
       ankane/pgvector
     ```
     If you already have an instance of `pgvector` you can create a database for Dewy and configure Dewy use it using the `DB` env var (see below).
-    
+
 1. Install Dewy
     ```
     pip install dewy
@@ -116,7 +116,7 @@ To get a local copy up and running follow these steps.
     ```typescript
     const context = await dewy.default.retrieveChunks({
       collection_id: 1,
-      query: "tell me about RAG", 
+      query: "tell me about RAG",
       n: 10,
     });
 
@@ -155,7 +155,7 @@ Don't see a feature that would make Dewy better for your application - [create a
 
 * Support more document formats (ie [Markdown](https://github.com/DewyKB/dewy/issues/29), [DOCX](https://github.com/DewyKB/dewy/issues/28), [HTML](https://github.com/DewyKB/dewy/issues/27))
 * Support more types of chunk extractors
-* Multi-modal search over images, tables, audio, etc. 
+* Multi-modal search over images, tables, audio, etc.
 * Integrations with LangChain, LlamaIndex, Haystack, etc.
 * Support flexible result ranking (ie rag-fusion, mmr, etc).
 * Provide metrics around which chunks are used, relevance scores, etc.
@@ -219,6 +219,18 @@ If you're in a `poetry shell`, you can omit the `poetry run`:
 * Type Checking: `poetry run mypy app`
 
 
+<p align="right">(<a href="#readme-top">back to top</a>)</p
+
+### Releasing
+
+1. Look at the [draft release](https://github.com/DewyKB/dewy/releases) to determine the suggested next version.
+2. Create a PR updating the following locations to that version:
+  a. [`pyproject.toml`](https://github.com/DewyKB/dewy/blob/main/pyproject.toml#L3) for `dewy`
+  b. API version in [`config.py`](https://github.com/DewyKB/dewy/blob/main/dewy/config.py#L69)
+  c. `openapi.yaml` and `dewy-client` by running `poe extract-openapi` and `poe update-client`.
+3. Once that PR is in, edit the draft release, make sure the version and tag match what you selected in step 1 (and used in the PR), check "Set as a pre-release" (will be updated by the release automation) and choose to publish the release.
+4. The release automation should kick in and work through the release steps. It will need approval for the pypi deployment environment to publish the `dewy` and `dewy-client` packages.
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p
 
 <!-- LICENSE -->