Script and CI action to validate backwards compatibility

.github/workflows/build.yml

@@ -62,3 +62,22 @@ jobs:
           python -m pessimist --fast --requirements= -c 'python -m usort --help' .
           python -m pessimist --fast --requirements=importall.txt --fast -c 'importall --root=. usort' .
 
+  backward-compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Set Up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Check backwards compatibility
+        shell: bash
+        run: |
+          python -m pip install -U pip packaging
+          make backcompat

Makefile

@@ -45,6 +45,10 @@ lint:
 deps:
 	python -m pessimist --requirements= -c "python -m usort --help" .
 
+.PHONY: backcompat
+backcompat:
+	python check_backcompat.py
+
 .PHONY: html
 html:
 	sphinx-build -ab html docs html

check_backcompat.py

@@ -0,0 +1,124 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Check the current repo against all previous supported versions of µsort
+
+If any previous version triggers sorting/formatting changes, we consider that
+worthy of a major version bump, and should be reverted or at least reconsidered.
+
+See the Versioning Guide for details.
+
+TODO: include other projects as part of testing, like black, libcst, etc
+"""
+
+import json
+import platform
+import subprocess
+import sys
+import venv
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import List, Optional
+from urllib.request import urlopen
+
+from packaging.version import Version
+
+REPO_ROOT = Path(__file__).parent.resolve()
+TARGET_VERSION = Version("1.0.0")
+PYPI_JSON_URL = "https://pypi.org/pypi/usort/json"
+
+
+def get_public_versions() -> List[Version]:
+    """
+    Find all non-yanked versions of usort >= TARGET_VERSION
+    """
+    with urlopen(PYPI_JSON_URL) as request:
+        data = json.loads(request.read())
+
+    versions: List[Version] = []
+    for version_str in data["releases"]:
+        version = Version(version_str)
+        if all(dist["yanked"] for dist in data["releases"][version_str]):
+            continue
+        if version >= TARGET_VERSION:
+            versions.append(version)
+
+    return sorted(versions, reverse=True)
+
+
+def setup_virtualenv(root: Path, version: Optional[Version] = None) -> Path:
+    """
+    Create venv, install usort, return path to `usort` binary
+    """
+
+    venv_path = root / f"venv-{version}" if version else root / "venv-local"
+    venv.create(venv_path, clear=True, with_pip=True)
+
+    bin_dir = (
+        venv_path / "Scripts" if platform.system() == "Windows" else venv_path / "bin"
+    )
+    python = bin_dir / "python"
+
+    subprocess.run((python, "-m", "pip", "-q", "install", "-U", "pip"), check=True)
+    if version:
+        subprocess.run(
+            (python, "-m", "pip", "-q", "install", "-U", f"usort=={version}"),
+            check=True,
+        )
+    else:
+        subprocess.run(
+            (python, "-m", "pip", "-q", "install", "-U", REPO_ROOT), check=True
+        )
+    return bin_dir / "usort"
+
+
+def check_versions(versions: List[Version]) -> List[Version]:
+    """
+    Format with local version, then check all released versions for regressions
+    """
+    with TemporaryDirectory() as td:
+        root = Path(td).resolve()
+
+        try:
+            print("sorting with local version ...")
+            usort = setup_virtualenv(root)
+            subprocess.run((usort, "--version"), check=True)
+            subprocess.run((usort, "format", "usort"), check=True)
+            print("done\n")
+        except Exception as e:
+            return [("local", e)]
+
+        failures: List[Version] = []
+        for version in versions:
+            try:
+                print(f"checking version {version} ...")
+                usort = setup_virtualenv(root, version)
+                subprocess.run((usort, "--version"), check=True)
+                subprocess.run((usort, "check", "usort"), check=True)
+                print("clean\n")
+            except Exception as e:
+                failures.append(version)
+
+        return failures
+
+
+def main() -> None:
+    versions = get_public_versions()
+    versions_str = ", ".join(str(v) for v in versions)
+    print(f"discovered versions {versions_str}\n")
+
+    failures = check_versions(versions)
+    if failures:
+        print("Sorting failed in versions:")
+        for version, exc in failures:
+            print(f"  {version}: {exc}")
+        sys.exit(1)
+    else:
+        print("success!")
+
+
+if __name__ == "__main__":
+    main()