Merge #380

380: Mechanized Release Process r=pathunstrom a=astronouth7303

Implements a mechanized release process. Under this, the release process would be:

1. Go to GitHub Release pages
2. Look at the diff since last version, compile changelog
3. On the GitHub site, create a tag/release with the changelog and any other release notes
4. Cirrus CI is responsible for building and uploading to PyPI and GitHub

Additionally, dev releases are uploaded to PyPI test every time `master` is updated.

Version numbers are no longer explicitly listed in the repo, instead being calculated based on the git tags.

This code is a pretty direct copy from https://github.com/gqlmod/gqlmod (which is in part based on code that @duckinator wrote).

@nbraud originally proposed this for PPB.

Co-authored-by: Jamie Bliss <jamie@ivyleav.es>

.ci/install-wheels.py

@@ -0,0 +1,22 @@
+import glob
+import io
+import os
+import ssl
+import subprocess
+from urllib.request import urlopen
+import zipfile
+
+CIRRUS_BUILD_ID = os.environ['CIRRUS_BUILD_ID']
+ARTIFACTS_URL = "https://api.cirrus-ci.com/v1/artifact/build/{}/build/dist.zip".format(CIRRUS_BUILD_ID)
+
+# Windows certificate validation fails. This is per PEP476
+with urlopen(ARTIFACTS_URL, context=ssl._create_unverified_context()) as resp:
+    zipdata = resp.read()
+
+with zipfile.ZipFile(io.BytesIO(zipdata)) as zf:
+    zf.extractall()
+
+subprocess.run(
+    ['pip', 'install'] + glob.glob('dist/*.whl'),
+    check=True
+)

.ci/upload.xsh

@@ -0,0 +1,65 @@
+#!/usr/bin/xonsh
+import io
+import sys
+import tempfile
+from urllib.request import urlopen, Request
+from urllib.error import HTTPError
+import zipfile
+
+$RAISE_SUBPROC_ERROR = True
+
+ARTIFACTS_URL = f"https://api.cirrus-ci.com/v1/artifact/build/{$CIRRUS_BUILD_ID}/build/dist.zip"
+PYPI_TEST_REPO = "https://test.pypi.org/legacy/"
+
+
+with tempfile.TemporaryDirectory() as td:
+    cd @(td)
+
+    with urlopen(ARTIFACTS_URL) as resp:
+        zipdata = resp.read()
+
+    with zipfile.ZipFile(io.BytesIO(zipdata)) as zf:
+        # nl = zf.namelist()
+        # print(f"Found {len(nl)} files from build:", *nl)
+        zf.extractall()
+
+    dists = [f for f in pg`**` if '+' not in f.name and f.is_file()]
+
+    if not dists:
+        print("No uploadable dists found, skipping upload")
+        sys.exit(0)
+    else:
+        print("Found dists:", *dists)
+
+    print("Uploading to test repo...")
+
+    twine upload --repository-url @(PYPI_TEST_REPO) --username __token__ --password $TWINE_TEST_TOKEN @(dists)
+
+    print("")
+
+    if 'CIRRUS_RELEASE' in ${...}:
+        print("Uploading to GitHub...")
+        for dist in dists:
+            print(f"\t{dist.name}...")
+            dest_url = f"https://uploads.github.com/repos/{$CIRRUS_REPO_FULL_NAME}/releases/{$CIRRUS_RELEASE}/assets?name={dist.name}"
+            with dist.open('rb') as fobj:
+                buff = fobj.read()
+                try:
+                    resp = urlopen(Request(
+                        url=dest_url,
+                        method='POST',
+                        data=buff,
+                        headers={
+                            "Authorization": f"token {$GITHUB_TOKEN}",
+                            "Content-Type": "application/octet-stream",
+                        },
+                    ))
+                except HTTPError as exc:
+                    print(exc.headers)
+                    print(exc.read())
+                    raise
+
+        print("")
+
+        print("Uploading to PyPI...")
+        twine upload --username __token__ --password $TWINE_PROD_TOKEN @(dists)

.cirrus.yml

@@ -1,51 +1,75 @@
-docs_task:
+pep517_task:
   skip: $CIRRUS_BRANCH =~ '.*\.tmp'
   container:
-    image: python:3.7-slim
+    image: python:3.7
 
   install_script:
-    - apt update || true ; apt install -qq -y make
-    - pip install --upgrade-strategy eager -U -r requirements-docs.txt
-    - pip install .
+    - pip install --upgrade-strategy eager -U pep517
 
   script:
-    - make -C docs/ linkcheck
-    - make -C docs/ html
+    - python3 -m pep517.check .
 
-pep517_task:
+
+build_task:
+  container:
+    image: python:3
+  setup_script:
+    - git fetch --tags
+    - pip install bork
+  script:
+    - bork build
+  dist_artifacts:
+    path: "dist/**"
+
+
+docs_task:
   skip: $CIRRUS_BRANCH =~ '.*\.tmp'
   container:
     image: python:3.7-slim
 
+  depends_on:
+    - build
+
   install_script:
-    - pip install --upgrade-strategy eager -U pep517
+    - apt update || true ; apt install -qq -y make
+    - pip install --upgrade-strategy eager -U -r requirements-docs.txt
+    - python .ci/install-wheels.py
 
   script:
-    - python3 -m pep517.check .
-
+    - make -C docs/ linkcheck
+    - make -C docs/ html
 
 task:
   name: "Linux $IMAGE"
+  alias: Tests
   skip: $CIRRUS_BRANCH =~ '.*\.tmp'
   allow_failures: $IMAGE =~ '.*-rc-.*'
   env:
     matrix:
       - IMAGE: python:3.6-slim
       - IMAGE: python:3.7-slim
-      # - IMAGE: python:3.8-rc-slim
+      # - IMAGE: python:3.8-slim
       - IMAGE: pypy:3.6-slim
 
   container:
     image: $IMAGE
 
+  depends_on:
+    - build
+
   install_script:
     - >-
       if command -v pypy3 >/dev/null || python3 -c 'import sys; exit(sys.implementation.version.releaselevel == "final")'; then
         apt update || true;
         apt install -qq -y pkgconf libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsdl1.2-dev libsmpeg-dev libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev libfreetype6-dev gcc
       fi
     - pip install --upgrade-strategy eager -U -r requirements-tests.txt
-    - pip install .
+    - >-
+      if command -v pypy3 >/dev/null; then
+        pypy3 .ci/install-wheels.py
+      else
+        python .ci/install-wheels.py
+      fi
 
   script:
     - command -v pypy3 >/dev/null && export PY=pypy3
@@ -54,6 +78,7 @@ task:
     - pytest
 
 macOS_task:
+  alias: Tests
   skip: $CIRRUS_BRANCH =~ '.*\.tmp'
   osx_instance:
     image: catalina-base
@@ -62,6 +87,8 @@ macOS_task:
     matrix:
       - PYTHON: 3.6.8
       - PYTHON: 3.7.2
+  depends_on:
+    - build
   install_script:
     # Per the pyenv homebrew recommendations.
     # https://github.com/pyenv/pyenv/wiki#suggested-build-environment
@@ -71,7 +98,7 @@ macOS_task:
     - pyenv global ${PYTHON}
     - pyenv rehash
     - pip install --upgrade-strategy eager -U -r requirements-tests.txt
-    - pip install .
+    - python .ci/install-wheels.py
 
   script:
     - python3 --version
@@ -80,6 +107,7 @@ macOS_task:
 
 task:
   name: "Windows $IMAGE"
+  alias: Tests
   skip: $CIRRUS_BRANCH =~ '.*\.tmp'
   allow_failures: $IMAGE =~ '.*-rc-.*'
   windows_container:
@@ -88,14 +116,37 @@ task:
 
   env:
     matrix:
-      - IMAGE: python:3.6-windowsservercore-1809
-      - IMAGE: python:3.7-windowsservercore-1809
+      - IMAGE: python:3.6-windowsservercore
+      - IMAGE: python:3.7-windowsservercore
 
   install_script:
     - C:\Python\python.exe -m pip install --upgrade-strategy eager -U -r requirements-tests.txt
-    - C:\Python\python.exe -m pip install .
+    - C:\Python\python.exe .ci/install-wheels.py
 
   script:
     - C:\Python\python.exe --version
     - C:\Python\python.exe -m pip list
     - C:\Python\python.exe -m pytest
+
+
+
+upload_task:
+  only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_RELEASE != ''
+  depends_on:
+    - build
+    - docs
+    - pep517
+    - Tests
+  env:
+    TWINE_TEST_TOKEN: "ENCRYPTED[4df6a9281bcbab67052d63d8ba4f5ee404d57fd95158e0e6e8693ac6b6ae0e2f2ec0ae8ac09a1fe9998b4675d6ccd7c2]"
+    TWINE_PROD_TOKEN: "ENCRYPTED[7ea667df4087e85f59575b59f07b2ef8da992831d45e0a0c60869aed64be252e4373b45b6a9f323a6a6365337bac0a7e]"
+    GITHUB_TOKEN: "ENCRYPTED[42fb2ca52f26c65b254620ce05eea27925844c034a0f73ca96f3fd4c80530a595e3c7dccda51df22b144a16673393704]"
+
+  container:
+    image: xonsh/xonsh:slim
+
+  install_script:
+    - pip install twine
+
+  script:
+    - xonsh .ci/upload.xsh

bors.toml

@@ -2,8 +2,8 @@ status = [
   "docs",
   "Linux python:3.6-slim",
   "Linux python:3.7-slim",
-  "Windows python:3.6-windowsservercore-1809",
-  "Windows python:3.7-windowsservercore-1809",
+  "Windows python:3.6-windowsservercore",
+  "Windows python:3.7-windowsservercore",
   "macOS PYTHON:3.6.8",
   "macOS PYTHON:3.7.2",
   "pep517",

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 # Minimum requirements for the build system to execute.
-requires = ["setuptools", "wheel"]  # PEP 508 specifications.
+requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm"]
 
 # Actually tell PEP517 tools to call setuptools
 build-backend = "setuptools.build_meta"

setup.cfg

@@ -3,7 +3,6 @@ test = pytest
 
 [metadata]
 name = ppb
-version = 0.7.0
 author = Piper Thunstrom
 author_email = pathunstrom@gmail.com
 description = An Event Driven Python Game Engine
@@ -32,5 +31,7 @@ packages =
 
 setup_requires =
     pytest-runner
+    wheel
+    setuptools_scm
 
 python_requires = >= 3.6, < 3.8

setup.py

@@ -15,6 +15,9 @@ def requirements(section=None):
 
 # See setup.cfg for the actual configuration.
 setup(
+    use_scm_version={
+        'local_scheme': 'dirty-tag',
+    },
     install_requires=requirements(),
     tests_require=requirements('tests'),
 )