[DNM] Python 3.9 + new version of toolchain plugin

build-support/bin/ci.py

@@ -60,6 +60,7 @@ def main() -> None:
 class PythonVersion(Enum):
     py37 = "3.7"
     py38 = "3.8"
+    py39 = "3.9"
 
     def __str__(self) -> str:
         return str(self.value)

build-support/bin/generate_travis_yml.py

@@ -79,8 +79,9 @@ def all_entries(cls) -> List[Dict[str, str]]:
     "BOOTSTRAPPED_PEX_KEY_PREFIX=daily/${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex",
     "NATIVE_ENGINE_SO_KEY_PREFIX=monthly/native_engine_so",
     "MACOS_PYENV_PY27_VERSION=2.7.18",
-    "MACOS_PYENV_PY37_VERSION=3.7.7",
-    "MACOS_PYENV_PY38_VERSION=3.8.3",
+    "MACOS_PYENV_PY37_VERSION=3.7.9",
+    "MACOS_PYENV_PY38_VERSION=3.8.7",
+    "MACOS_PYENV_PY39_VERSION=3.9.1",
     # NB: We must set `PYENV_ROOT` on macOS for Pyenv to work properly. However, on Linux, we must not
     # override the default value because Linux pre-installs Python via Pyenv and we must keep their
     # $PYENV_ROOT for this to still work.
@@ -114,22 +115,31 @@ def all_entries(cls) -> List[Dict[str, str]]:
 class PythonVersion(Enum):
     py37 = "py37"
     py38 = "py38"
+    py39 = "py39"
 
     def __str__(self) -> str:
         return str(self.value)
 
     @property
     def number(self) -> int:
-        return {self.py37: 37, self.py38: 38}[self]  # type: ignore[index]
+        return {self.py37: 37, self.py38: 38, self.py39: 39}[self]  # type: ignore[index]
 
     @property
     def decimal(self) -> float:
-        return {self.py37: 3.7, self.py38: 3.8}[self]  # type: ignore[index]
+        return {self.py37: 3.7, self.py38: 3.8, self.py39: 3.9}[self]  # type: ignore[index]
 
     def default_stage(self, *, is_bootstrap: bool = False) -> Stage:
         if is_bootstrap:
-            return {self.py37: Stage.bootstrap, self.py38: Stage.bootstrap_cron}[self]  # type: ignore[index]
-        return {self.py37: Stage.test, self.py38: Stage.test_cron}[self]  # type: ignore[index]
+            return {
+                self.py37: Stage.bootstrap,
+                self.py38: Stage.bootstrap_cron,
+                self.py39: Stage.bootstrap_cron,
+            }[
+                self  # type: ignore[index]
+            ]
+        return {self.py37: Stage.test, self.py38: Stage.test_cron, self.py39: Stage.test_cron}[
+            self  # type: ignore[index]
+        ]
 
 
 # ----------------------------------------------------------------------
@@ -272,7 +282,7 @@ def _linux_before_install(
         # TODO(John Sirois): Get rid of this in favor of explicitly adding pyenv versions to the PATH:
         #   https://github.com/pantsbuild/pants/issues/7601
         (
-            "pyenv global 2.7.17 3.6.10 3.7.6 3.8.1"
+            "pyenv global 2.7.17 3.6.10 3.7.6 3.8.1 3.9.1"
             if not xenial
             else "pyenv global 2.7.15 3.6.7 3.7.1"
         ),
@@ -305,7 +315,7 @@ def linux_shard(
     setup = {
         "os": "linux",
         "dist": "bionic",
-        "python": ["2.7", "3.6", "3.7"],
+        "python": "3.9",
         "addons": {
             "apt": {
                 "packages": [
@@ -385,6 +395,7 @@ def _osx_env_with_pyenv() -> List[str]:
         'PATH="${PYENV_ROOT}/versions/${MACOS_PYENV_PY27_VERSION}/bin:${PATH}"',
         'PATH="${PYENV_ROOT}/versions/${MACOS_PYENV_PY37_VERSION}/bin:${PATH}"',
         'PATH="${PYENV_ROOT}/versions/${MACOS_PYENV_PY38_VERSION}/bin:${PATH}"',
+        'PATH="${PYENV_ROOT}/versions/${MACOS_PYENV_PY39_VERSION}/bin:${PATH}"',
     ]
 
 
@@ -555,6 +566,7 @@ def _build_wheels_command(homedir: str = "${HOME}") -> List[str]:
         *_install_rust(homedir=homedir),
         "./build-support/bin/release.sh -n",
         "USE_PY38=true ./build-support/bin/release.sh -n",
+        "USE_PY39=true ./build-support/bin/release.sh -n",
         # NB: We also build `fs_util` in this shard to leverage having had compiled the engine.
         "./build-support/bin/release.sh -f",
     ]
@@ -570,6 +582,7 @@ def build_wheels_linux() -> Dict:
         **CACHE_NATIVE_ENGINE,
         **linux_shard(use_docker=True),
         "name": "Build Linux wheels and fs_util",
+        "stage": Stage.test.value,
         "script": [docker_build_travis_ci_image(), docker_run_travis_ci_image(command)],
         "if": SKIP_WHEELS_CONDITION,
     }
@@ -582,6 +595,7 @@ def build_wheels_osx() -> Dict:
         **CACHE_NATIVE_ENGINE,
         **osx_shard(osx_image="xcode8"),
         "name": "Build macOS wheels and fs_util",
+        "stage": Stage.test.value,
         "script": _build_wheels_command(),
         "before_install": _osx_before_install(
             python_versions=[PythonVersion.py37, PythonVersion.py38],
@@ -779,7 +793,7 @@ def ignore_aliases(self, data):
 
 
 def main() -> None:
-    supported_python_versions = [PythonVersion.py37, PythonVersion.py38]
+    supported_python_versions = [PythonVersion.py37, PythonVersion.py38, PythonVersion.py39]
     generated_yaml = yaml.dump(
         {
             # Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1

build-support/bin/packages.py

@@ -433,10 +433,10 @@ def check_prebuilt_wheels(check_dir: str) -> None:
         if not local_files:
             missing_packages.append(package.name)
             continue
-        if is_cross_platform(local_files) and len(local_files) != 4:
+        if is_cross_platform(local_files) and len(local_files) != 6:
             formatted_local_files = ", ".join(f.name for f in local_files)
             missing_packages.append(
-                f"{package.name} (expected 4 wheels, {{macosx, linux}} x {{cp37m, cp38}}, "
+                f"{package.name} (expected 6 wheels, {{macosx, linux}} x {{cp37m, cp38, cp39}}, "
                 f"but found {formatted_local_files})"
             )
     if missing_packages:

build-support/bin/release.sh

@@ -26,6 +26,9 @@ source "${ROOT}/build-support/common.sh"
 if [[ "${USE_PY38:-false}" == "true" ]]; then
   default_python=python3.8
   interpreter_constraint="==3.8.*"
+elif [[ "${USE_PY39:-false}" == "true" ]]; then
+  default_python=python3.9
+  interpreter_constraint="==3.9.*"
 else
   default_python=python3.7
   interpreter_constraint="==3.7.*"
@@ -36,8 +39,8 @@ if ! command -v "${PY}" >/dev/null; then
   die "Python interpreter ${PY} not discoverable on your PATH."
 fi
 py_major_minor=$(${PY} -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
-if [[ "${py_major_minor}" != "3.7" && "${py_major_minor}" != "3.8" ]]; then
-  die "Invalid interpreter. The release script requires Python 3.7 or 3.8 (you are using ${py_major_minor})."
+if [[ "${py_major_minor}" != "3.7" && "${py_major_minor}" != "3.8" && "${py_major_minor}" != "3.9" ]]; then
+  die "Invalid interpreter. The release script requires Python 3.7, 3.8, or 3.9 (you are using ${py_major_minor})."
 fi
 
 # This influences what setuptools is run with, which determines the interpreter used for building
@@ -334,7 +337,7 @@ function build_pex() {
       ;;
     fetch)
       local distribution_target_flags=()
-      abis=("cp-37-m" "cp-38-cp38")
+      abis=("cp-37-m" "cp-38-cp38" "cp-39-cp39")
       for platform in "${linux_platform_noabi}" "${osx_platform_noabi}"; do
         for abi in "${abis[@]}"; do
           distribution_target_flags=("${distribution_target_flags[@]}" "--platform=${platform}-${abi}")

build-support/common.sh

@@ -74,7 +74,7 @@ function determine_python() {
     echo "${PY}"
     return 0
   fi
-  for version in '3.7' '3.8'; do
+  for version in '3.7' '3.8' '3.9'; do
     local interpreter_path
     interpreter_path="$(command -v "python${version}")"
     if [[ -z "${interpreter_path}" ]]; then

pants.toml

@@ -14,14 +14,14 @@ backend_packages.add = [
   "pants.backend.python.typecheck.mypy",
   "pants.backend.python.mixed_interpreter_constraints",
   "internal_plugins.releases",
-#  "toolchain.pants.auth",
-#  "toolchain.pants.buildsense",
-#  "toolchain.pants.common",
+  "toolchain.pants.auth",
+  "toolchain.pants.buildsense",
+  "toolchain.pants.common",
 ]
 
-#plugins = [
-#  "toolchain.pants.plugin==0.3.0",
-#]
+plugins = [
+  "toolchain.pants.plugin==0.4.0",
+]
 
 build_file_prelude_globs = ["pants-plugins/python_integration_tests_macro.py"]
 
@@ -62,7 +62,7 @@ root_patterns = [
 resolver_version = "pip-2020-resolver"
 requirement_constraints = "3rdparty/python/constraints.txt"
 resolve_all_constraints = "nondeployables"
-interpreter_constraints = [">=3.7,<3.9"]
+interpreter_constraints = [">=3.7,<3.10"]
 
 [black]
 config = "pyproject.toml"
@@ -137,10 +137,10 @@ pytest_plugins.add = [
 timeout_default = 60
 
 [coverage-py]
-interpreter_constraints = [">=3.7,<3.9"]
+interpreter_constraints = [">=3.7,<3.10"]
 
 [sourcefile-validation]
 config = "@build-support/regexes/config.yaml"
 
-#[toolchain-setup]
-#repo = "pants"
+[toolchain-setup]
+repo = "pants"

pants.travis-ci.toml

@@ -17,10 +17,10 @@ junit_xml_dir = "dist/test-results/"
 [coverage-py]
 report = ["raw", "xml"]
 
-#[auth]
-#from_env_var = "TOOLCHAIN_AUTH_TOKEN"
-#org = "pantsbuild"
-#ci_env_variables = ["TRAVIS", "TRAVIS_JOB_ID", "TRAVIS_BUILD_ID", "TRAVIS_PULL_REQUEST", "TRAVIS_BUILD_WEB_URL"]
-#
-#[buildsense]
-#enable = true
+[auth]
+from_env_var = "TOOLCHAIN_AUTH_TOKEN"
+org = "pantsbuild"
+ci_env_variables = ["TRAVIS", "TRAVIS_JOB_ID", "TRAVIS_BUILD_ID", "TRAVIS_PULL_REQUEST", "TRAVIS_BUILD_WEB_URL"]
+
+[buildsense]
+enable = true

src/python/pants/option/optionable_test.py

@@ -13,7 +13,9 @@ class NoScope(Optionable):
 
         with self.assertRaises(TypeError) as cm:
             NoScope()  # type: ignore[abstract]
-        assert ("with abstract methods options_scope") in str(cm.exception)
+        assert "with abstract methods options_scope" in str(
+            cm.exception
+        ) or "with abstract method options_scope" in str(cm.exception)
 
         class StringScope(Optionable):
             options_scope = "good"

src/python/pants/testutil/BUILD

@@ -8,7 +8,7 @@ python_distribution(
     ':int-test-for-export',
     ':rule_runner',
   ],
-  setup_py_commands=["bdist_wheel", "--python-tag", "py37.py38", "sdist"],
+  setup_py_commands=["bdist_wheel", "--python-tag", "py37.py38.py39", "sdist"],
   provides=setup_py(
     name='pantsbuild.pants.testutil',
     description='Test support for writing Pants plugins.',