diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 46ad6cd981f5ae..3d190242dd7d15 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -217,11 +217,10 @@ jobs:
       - name: "Build wheels"
         uses: PyO3/maturin-action@v1
         with:
-          manylinux: auto
           args: --out dist
       - name: "Test wheel"
         run: |
-          pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall
+          pip install --force-reinstall --find-links dist ${{ env.PACKAGE_NAME }}
           ruff --help
           python -m ruff --help
       - name: "Remove wheels from cache"
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index d45dcfdffe8628..0c1fae26aae88d 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -2,8 +2,13 @@ name: "[ruff] Release"
 
 on:
   workflow_dispatch:
-  release:
-    types: [ published ]
+    tag:
+      description: "The version to tag, without the leading 'v'"
+      type: string
+  push:
+    paths:
+      # When we change pyproject.toml, we want to ensure that the maturin builds still work
+      - pyproject.toml
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -394,7 +399,8 @@ jobs:
       - linux-cross
       - musllinux
       - musllinux-cross
-    if: "startsWith(github.ref, 'refs/tags/')"
+    # If you don't set an input it's a practice run
+    if: "${{ inputs.tag }} != ''"
     environment:
       name: release
     permissions:
@@ -403,11 +409,19 @@ jobs:
       # For GitHub release publishing
       contents: write
     steps:
+      - name: Consistency check
+        run: |
+          version=$(grep "version = " pyproject.toml | sed -e 's/version = "\(.*\)"/\1/g')
+          echo "${{ inputs.tag }}"
+          echo "${version}"
+          if [ "${{ inputs.tag }}" != "${version}" ]; then
+            exit 1
+          fi
       - uses: actions/download-artifact@v3
         with:
           name: wheels
           path: wheels
-      - name: "Publish to PyPi"
+      - name: Publish to PyPi
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           skip-existing: true
@@ -417,10 +431,17 @@ jobs:
         with:
           name: binaries
           path: binaries
+      - name: git tag
+        run: |
+          git tag -m "v${{ inputs.tag }}" "v${{ inputs.tag }}"
+          # If there is duplicate tag, this will fail. The publish to pypi action will have been a noop (due to skip
+          # existing), so we make a non-destructive exit here
+          git push --tags
       - name: "Publish to GitHub"
         uses: softprops/action-gh-release@v1
         with:
           files: binaries/*
+          tag_name: v${{ inputs.tag }}
 
   # After the release has been published, we update downstream repositories
   # This is separate because if this fails the release is still fine, we just need to do some manual workflow triggers
diff --git a/How_to_Release.md b/How_to_Release.md
new file mode 100644
index 00000000000000..8285ab42c2d457
--- /dev/null
+++ b/How_to_Release.md
@@ -0,0 +1,18 @@
+# Creating a new release
+
+This is a guide how to create a new ruff release
+
+1. Update the version with `rg 0.0.269 --files-with-matches | xargs sed -i 's/0.0.269/0.0.270/g'`
+2. Update [BREAKING_CHANGES.md](BREAKING_CHANGES.md)
+3. Create a PR with the version and BREAKING_CHANGES.md updated
+4. Merge the PR
+5. Run the release workflow with the tag (without starting `v`) as input. Make sure main has your merged PR as last commit
+6. The release workflow will do the following:
+   1. Build all the assets. If this fails (even though we tested in step 4), we haven’t tagged or uploaded anything, you can restart after pushing a fix
+   2. Upload to pypi
+   3. Create a git tag (from pyproject.toml) and push git tag. We create the git tag only here because we can't change it ([#4468](https://github.com/charliermarsh/ruff/issues/4468)), so we want to make sure everything up to and including publishing to pypi worked
+   4. Attach artifacts to draft GitHub release
+   5. Trigger downstream repositories. This can fail without causing fallout, it is possible (if inconvenient) to trigger the downstream jobs manually
+7. Create release notes in GitHub UI (https://github.com/charliermarsh/ruff/releases/new)
+8. If needed, [update the schemastore](https://github.com/charliermarsh/ruff/blob/main/scripts/update_schemastore.py)
+9. If needed, update ruff-lsp and ruff-vscode