Merge pull request #3065 from jsiirola/manage-pyutilib-imports

Resolve testing in Python 3.12 (manage `pyutilib`, `distutils` dependencies)

.github/workflows/test_branches.yml

@@ -592,8 +592,8 @@ jobs:
         echo "COVERAGE_PROCESS_START=$COVERAGE_RC" >> $GITHUB_ENV
         cp ${GITHUB_WORKSPACE}/.coveragerc ${COVERAGE_RC}
         echo "data_file=${COVERAGE_BASE}age" >> ${COVERAGE_RC}
-        SITE_PACKAGES=$($PYTHON_EXE -c "from distutils.sysconfig import \
-            get_python_lib; print(get_python_lib())")
+        SITE_PACKAGES=$($PYTHON_EXE -c \
+            "import sysconfig; print(sysconfig.get_path('purelib'))")
         echo "Python site-packages: $SITE_PACKAGES"
         echo 'import coverage; coverage.process_startup()' \
             > ${SITE_PACKAGES}/run_coverage_at_startup.pth

.github/workflows/test_pr_and_main.yml

@@ -622,8 +622,8 @@ jobs:
         echo "COVERAGE_PROCESS_START=$COVERAGE_RC" >> $GITHUB_ENV
         cp ${GITHUB_WORKSPACE}/.coveragerc ${COVERAGE_RC}
         echo "data_file=${COVERAGE_BASE}age" >> ${COVERAGE_RC}
-        SITE_PACKAGES=$($PYTHON_EXE -c "from distutils.sysconfig import \
-            get_python_lib; print(get_python_lib())")
+        SITE_PACKAGES=$($PYTHON_EXE -c \
+            "import sysconfig; print(sysconfig.get_path('purelib'))")
         echo "Python site-packages: $SITE_PACKAGES"
         echo 'import coverage; coverage.process_startup()' \
             > ${SITE_PACKAGES}/run_coverage_at_startup.pth

.jenkins.sh

@@ -77,7 +77,7 @@ if test -z "$MODE" -o "$MODE" == setup; then
     source python/bin/activate
     # Because modules set the PYTHONPATH, we need to make sure that the
     # virtualenv appears first
-    LOCAL_SITE_PACKAGES=`python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"`
+    LOCAL_SITE_PACKAGES=`python -c "import sysconfig; print(sysconfig.get_path('purelib'))"`
     export PYTHONPATH="$LOCAL_SITE_PACKAGES:$PYTHONPATH"
 
     # Set up Pyomo checkouts

pyomo/common/cmake_builder.py

@@ -32,11 +32,8 @@ def handleReadonly(function, path, excinfo):
 def build_cmake_project(
     targets, package_name=None, description=None, user_args=[], parallel=None
 ):
-    # Note: setuptools must be imported before distutils to avoid
-    # warnings / errors with recent setuptools distributions
-    from setuptools import Extension
-    import distutils.core
-    from distutils.command.build_ext import build_ext
+    from setuptools import Extension, Distribution
+    from setuptools.command.build_ext import build_ext
 
     class _CMakeBuild(build_ext, object):
         def run(self):
@@ -122,7 +119,7 @@ def __init__(self, target_dir, user_args, parallel):
         'ext_modules': ext_modules,
         'cmdclass': {'build_ext': _CMakeBuild},
     }
-    dist = distutils.core.Distribution(package_config)
+    dist = Distribution(package_config)
     basedir = os.path.abspath(os.path.curdir)
     try:
         tmpdir = os.path.abspath(tempfile.mkdtemp())

pyomo/common/dependencies.py

@@ -826,14 +826,23 @@ def _finalize_numpy(np, available):
         numeric_types.RegisterComplexType(t)
 
 
+def _pyutilib_importer():
+    # On newer Pythons, PyUtilib import will fail, but only if a
+    # second-level module is imported.  We will arbitrarily choose to
+    # check pyutilib.component (as that is the path exercised by the
+    # pyomo.common.tempfiles deprecation path)
+    importlib.import_module('pyutilib.component')
+    return importlib.import_module('pyutilib')
+
+
 dill, dill_available = attempt_import('dill')
 mpi4py, mpi4py_available = attempt_import('mpi4py')
 networkx, networkx_available = attempt_import('networkx')
 numpy, numpy_available = attempt_import('numpy', callback=_finalize_numpy)
 pandas, pandas_available = attempt_import('pandas')
 plotly, plotly_available = attempt_import('plotly')
 pympler, pympler_available = attempt_import('pympler', callback=_finalize_pympler)
-pyutilib, pyutilib_available = attempt_import('pyutilib')
+pyutilib, pyutilib_available = attempt_import('pyutilib', importer=_pyutilib_importer)
 scipy, scipy_available = attempt_import(
     'scipy',
     callback=_finalize_scipy,

pyomo/common/tempfiles.py

@@ -22,18 +22,15 @@
 import logging
 import shutil
 import weakref
+
+from pyomo.common.dependencies import attempt_import, pyutilib_available
 from pyomo.common.deprecation import deprecated, deprecation_warning
 from pyomo.common.errors import TempfileContextError
 from pyomo.common.multithread import MultiThreadWrapperWithMain
 
-try:
-    from pyutilib.component.config.tempfiles import TempfileManager as pyutilib_mngr
-except ImportError:
-    pyutilib_mngr = None
-
 deletion_errors_are_fatal = True
-
 logger = logging.getLogger(__name__)
+pyutilib_tempfiles, _ = attempt_import('pyutilib.component.config.tempfiles')
 
 
 class TempfileManagerClass(object):
@@ -432,16 +429,17 @@ def _resolve_tempdir(self, dir=None):
             return self.manager().tempdir
         elif TempfileManager.main_thread.tempdir is not None:
             return TempfileManager.main_thread.tempdir
-        elif pyutilib_mngr is not None and pyutilib_mngr.tempdir is not None:
-            deprecation_warning(
-                "The use of the PyUtilib TempfileManager.tempdir "
-                "to specify the default location for Pyomo "
-                "temporary files has been deprecated.  "
-                "Please set TempfileManager.tempdir in "
-                "pyomo.common.tempfiles",
-                version='5.7.2',
-            )
-            return pyutilib_mngr.tempdir
+        elif pyutilib_available:
+            if pyutilib_tempfiles.TempfileManager.tempdir is not None:
+                deprecation_warning(
+                    "The use of the PyUtilib TempfileManager.tempdir "
+                    "to specify the default location for Pyomo "
+                    "temporary files has been deprecated.  "
+                    "Please set TempfileManager.tempdir in "
+                    "pyomo.common.tempfiles",
+                    version='5.7.2',
+                )
+                return pyutilib_tempfiles.TempfileManager.tempdir
         return None
 
     def _remove_filesystem_object(self, name):

pyomo/common/tests/test_tempfile.py

@@ -30,18 +30,14 @@
 
 import pyomo.common.tempfiles as tempfiles
 
+from pyomo.common.dependencies import pyutilib_available
 from pyomo.common.log import LoggingIntercept
 from pyomo.common.tempfiles import (
     TempfileManager,
     TempfileManagerClass,
     TempfileContextError,
 )
 
-try:
-    from pyutilib.component.config.tempfiles import TempfileManager as pyutilib_mngr
-except ImportError:
-    pyutilib_mngr = None
-
 old_tempdir = TempfileManager.tempdir
 tempdir = None
 
@@ -528,13 +524,13 @@ def test_open_tempfile_windows(self):
             f.close()
             os.remove(fname)
 
-    @unittest.skipIf(pyutilib_mngr is None, "deprecation test requires pyutilib")
+    @unittest.skipUnless(pyutilib_available, "deprecation test requires pyutilib")
     def test_deprecated_tempdir(self):
         self.TM.push()
         try:
             tmpdir = self.TM.create_tempdir()
-            _orig = pyutilib_mngr.tempdir
-            pyutilib_mngr.tempdir = tmpdir
+            _orig = tempfiles.pyutilib_tempfiles.TempfileManager.tempdir
+            tempfiles.pyutilib_tempfiles.TempfileManager.tempdir = tmpdir
             self.TM.tempdir = None
 
             with LoggingIntercept() as LOG:
@@ -556,7 +552,7 @@ def test_deprecated_tempdir(self):
             )
         finally:
             self.TM.pop()
-            pyutilib_mngr.tempdir = _orig
+            tempfiles.pyutilib_tempfiles.TempfileManager.tempdir = _orig
 
     def test_context(self):
         with self.assertRaisesRegex(

pyomo/contrib/appsi/build.py

@@ -63,8 +63,7 @@ def get_appsi_extension(in_setup=False, appsi_root=None):
 
 def build_appsi(args=[]):
     print('\n\n**** Building APPSI ****')
-    import setuptools
-    from distutils.dist import Distribution
+    from setuptools import Distribution
     from pybind11.setup_helpers import build_ext
     import pybind11.setup_helpers
     from pyomo.common.envvar import PYOMO_CONFIG_DIR

pyomo/contrib/mcpp/build.py

@@ -64,8 +64,8 @@ def _generate_configuration():
 
 
 def build_mcpp():
-    import distutils.core
-    from distutils.command.build_ext import build_ext
+    from setuptools import Distribution
+    from setuptools.command.build_ext import build_ext
 
     class _BuildWithoutPlatformInfo(build_ext, object):
         # Python3.x puts platform information into the generated SO file
@@ -87,7 +87,7 @@ def get_ext_filename(self, ext_name):
     print("\n**** Building MCPP library ****")
     package_config = _generate_configuration()
     package_config['cmdclass'] = {'build_ext': _BuildWithoutPlatformInfo}
-    dist = distutils.core.Distribution(package_config)
+    dist = Distribution(package_config)
     install_dir = os.path.join(envvar.PYOMO_CONFIG_DIR, 'lib')
     dist.get_command_obj('install_lib').install_dir = install_dir
     try:

pyomo/dataportal/plugins/__init__.py

@@ -9,8 +9,6 @@
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 
-from pyomo.common.dependencies import pyutilib, pyutilib_available
-
 
 def load():
     import pyomo.dataportal.plugins.csv_table
@@ -19,6 +17,4 @@ def load():
     import pyomo.dataportal.plugins.json_dict
     import pyomo.dataportal.plugins.text
     import pyomo.dataportal.plugins.xml_table
-
-    if pyutilib_available:
-        import pyomo.dataportal.plugins.sheet
+    import pyomo.dataportal.plugins.sheet

pyomo/dataportal/plugins/sheet.py

@@ -18,9 +18,18 @@
 # )
 from pyomo.dataportal.factory import DataManagerFactory
 from pyomo.common.errors import ApplicationError
-from pyomo.common.dependencies import attempt_import
+from pyomo.common.dependencies import attempt_import, importlib, pyutilib
 
-spreadsheet, spreadsheet_available = attempt_import('pyutilib.excel.spreadsheet')
+
+def _spreadsheet_importer():
+    # verify pyutilib imported correctly the first time
+    pyutilib.component
+    return importlib.import_module('pyutilib.excel.spreadsheet')
+
+
+spreadsheet, spreadsheet_available = attempt_import(
+    'pyutilib.excel.spreadsheet', importer=_spreadsheet_importer
+)
 
 
 def _attempt_open_excel():

pyomo/environ/tests/test_environ.py

@@ -168,7 +168,6 @@ def test_tpl_import_time(self):
         }
         # Non-standard-library TPLs that Pyomo will load unconditionally
         ref.add('ply')
-        ref.add('pyutilib')
         if numpy_available:
             ref.add('numpy')
         diff = set(_[0] for _ in tpl_by_time[-5:]).difference(ref)

setup.py

@@ -19,8 +19,10 @@
 from setuptools import setup, find_packages, Command
 
 try:
+    # This works beginning in setuptools 40.7.0 (27 Jan 2019)
     from setuptools import DistutilsOptionError
 except ImportError:
+    # Needed for setuptools prior to 40.7.0
     from distutils.errors import DistutilsOptionError