From eed6c7d6ed29083547a6bd04fc71042a4b79298a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 09:50:14 -0700 Subject: [PATCH 1/9] Leverage attempt_import to guard pyutilib tempfile dependency --- pyomo/common/tempfiles.py | 34 +++++++++++++++-------------- pyomo/common/tests/test_tempfile.py | 8 ++----- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/pyomo/common/tempfiles.py b/pyomo/common/tempfiles.py index e981d26d84e..0a38eac28d3 100644 --- a/pyomo/common/tempfiles.py +++ b/pyomo/common/tempfiles.py @@ -22,18 +22,15 @@ import logging import shutil import weakref + +from pyomo.common.dependencies import pyutilib_available from pyomo.common.deprecation import deprecated, deprecation_warning from pyomo.common.errors import TempfileContextError from pyomo.common.multithread import MultiThreadWrapperWithMain -try: - from pyutilib.component.config.tempfiles import TempfileManager as pyutilib_mngr -except ImportError: - pyutilib_mngr = None - deletion_errors_are_fatal = True - logger = logging.getLogger(__name__) +pyutilib_mngr = None class TempfileManagerClass(object): @@ -432,16 +429,21 @@ def _resolve_tempdir(self, dir=None): return self.manager().tempdir elif TempfileManager.main_thread.tempdir is not None: return TempfileManager.main_thread.tempdir - elif pyutilib_mngr is not None and pyutilib_mngr.tempdir is not None: - deprecation_warning( - "The use of the PyUtilib TempfileManager.tempdir " - "to specify the default location for Pyomo " - "temporary files has been deprecated. " - "Please set TempfileManager.tempdir in " - "pyomo.common.tempfiles", - version='5.7.2', - ) - return pyutilib_mngr.tempdir + elif pyutilib_available: + if pyutilib_mngr is None: + from pyutilib.component.config.tempfiles import ( + TempfileManager as pyutilib_mngr, + ) + if pyutilib_mngr.tempdir is not None: + deprecation_warning( + "The use of the PyUtilib TempfileManager.tempdir " + "to specify the default location for Pyomo " + "temporary files has been deprecated. " + "Please set TempfileManager.tempdir in " + "pyomo.common.tempfiles", + version='5.7.2', + ) + return pyutilib_mngr.tempdir return None def _remove_filesystem_object(self, name): diff --git a/pyomo/common/tests/test_tempfile.py b/pyomo/common/tests/test_tempfile.py index b82082ac1af..b549ed14cec 100644 --- a/pyomo/common/tests/test_tempfile.py +++ b/pyomo/common/tests/test_tempfile.py @@ -30,6 +30,7 @@ import pyomo.common.tempfiles as tempfiles +from pyomo.common.dependencies import pyutilib_available from pyomo.common.log import LoggingIntercept from pyomo.common.tempfiles import ( TempfileManager, @@ -37,11 +38,6 @@ TempfileContextError, ) -try: - from pyutilib.component.config.tempfiles import TempfileManager as pyutilib_mngr -except ImportError: - pyutilib_mngr = None - old_tempdir = TempfileManager.tempdir tempdir = None @@ -528,7 +524,7 @@ def test_open_tempfile_windows(self): f.close() os.remove(fname) - @unittest.skipIf(pyutilib_mngr is None, "deprecation test requires pyutilib") + @unittest.skipUnless(pyutilib_available, "deprecation test requires pyutilib") def test_deprecated_tempdir(self): self.TM.push() try: From 40764edf0e27f2ac7c4eabbe12d37b4599e2ea5a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 09:50:27 -0700 Subject: [PATCH 2/9] Remove pyutilib as an expected import in pyomo.environ --- pyomo/environ/tests/test_environ.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/environ/tests/test_environ.py b/pyomo/environ/tests/test_environ.py index 27a9f10cc08..02e4d723145 100644 --- a/pyomo/environ/tests/test_environ.py +++ b/pyomo/environ/tests/test_environ.py @@ -168,7 +168,6 @@ def test_tpl_import_time(self): } # Non-standard-library TPLs that Pyomo will load unconditionally ref.add('ply') - ref.add('pyutilib') if numpy_available: ref.add('numpy') diff = set(_[0] for _ in tpl_by_time[-5:]).difference(ref) From 52f39a16ee700c3524df6e9a107dbf2dc792c250 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 10:27:51 -0700 Subject: [PATCH 3/9] Resolve conflict between module attributes and imports --- pyomo/common/tempfiles.py | 12 ++++-------- pyomo/common/tests/test_tempfile.py | 6 +++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pyomo/common/tempfiles.py b/pyomo/common/tempfiles.py index 0a38eac28d3..f51fad3f3ac 100644 --- a/pyomo/common/tempfiles.py +++ b/pyomo/common/tempfiles.py @@ -23,14 +23,14 @@ import shutil import weakref -from pyomo.common.dependencies import pyutilib_available +from pyomo.common.dependencies import attempt_import, pyutilib_available from pyomo.common.deprecation import deprecated, deprecation_warning from pyomo.common.errors import TempfileContextError from pyomo.common.multithread import MultiThreadWrapperWithMain deletion_errors_are_fatal = True logger = logging.getLogger(__name__) -pyutilib_mngr = None +pyutilib_tempfiles, _ = attempt_import('pyutilib.component.config.tempfiles') class TempfileManagerClass(object): @@ -430,11 +430,7 @@ def _resolve_tempdir(self, dir=None): elif TempfileManager.main_thread.tempdir is not None: return TempfileManager.main_thread.tempdir elif pyutilib_available: - if pyutilib_mngr is None: - from pyutilib.component.config.tempfiles import ( - TempfileManager as pyutilib_mngr, - ) - if pyutilib_mngr.tempdir is not None: + if pyutilib_tempfiles.TempfileManager.tempdir is not None: deprecation_warning( "The use of the PyUtilib TempfileManager.tempdir " "to specify the default location for Pyomo " @@ -443,7 +439,7 @@ def _resolve_tempdir(self, dir=None): "pyomo.common.tempfiles", version='5.7.2', ) - return pyutilib_mngr.tempdir + return pyutilib_tempfiles.TempfileManager.tempdir return None def _remove_filesystem_object(self, name): diff --git a/pyomo/common/tests/test_tempfile.py b/pyomo/common/tests/test_tempfile.py index b549ed14cec..5e75c55305a 100644 --- a/pyomo/common/tests/test_tempfile.py +++ b/pyomo/common/tests/test_tempfile.py @@ -529,8 +529,8 @@ def test_deprecated_tempdir(self): self.TM.push() try: tmpdir = self.TM.create_tempdir() - _orig = pyutilib_mngr.tempdir - pyutilib_mngr.tempdir = tmpdir + _orig = tempfiles.pyutilib_tempfiles.TempfileManager.tempdir + tempfiles.pyutilib_tempfiles.TempfileManager.tempdir = tmpdir self.TM.tempdir = None with LoggingIntercept() as LOG: @@ -552,7 +552,7 @@ def test_deprecated_tempdir(self): ) finally: self.TM.pop() - pyutilib_mngr.tempdir = _orig + tempfiles.pyutilib_tempfiles.TempfileManager.tempdir = _orig def test_context(self): with self.assertRaisesRegex( From ab26ce7899452b65d7d8e234be5af127f31090d5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 11:41:52 -0700 Subject: [PATCH 4/9] Make the pyutilib import checker more robust for python3.12 failures --- pyomo/common/dependencies.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index a0717dba883..2ce12dcac0f 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -826,6 +826,17 @@ def _finalize_numpy(np, available): numeric_types.RegisterComplexType(t) +def _pyutilib_importer(): + # On newer Pythons, PyUtilib import will fail, but only if a + # second-level module is imported. We will arbirtarily choose to + # check pyutilib.component (as that is the path exercised by the + # pyomo.common.tempfiles deprecation path) + import pyutilib + import pyutilib.component + + return pyutilib + + dill, dill_available = attempt_import('dill') mpi4py, mpi4py_available = attempt_import('mpi4py') networkx, networkx_available = attempt_import('networkx') @@ -833,7 +844,7 @@ def _finalize_numpy(np, available): pandas, pandas_available = attempt_import('pandas') plotly, plotly_available = attempt_import('plotly') pympler, pympler_available = attempt_import('pympler', callback=_finalize_pympler) -pyutilib, pyutilib_available = attempt_import('pyutilib') +pyutilib, pyutilib_available = attempt_import('pyutilib', importer=_pyutilib_importer) scipy, scipy_available = attempt_import( 'scipy', callback=_finalize_scipy, From 407e3a2fb3cd2c33af6372ad83890be0a81f4628 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 11:52:51 -0700 Subject: [PATCH 5/9] NFC: fix typo --- pyomo/common/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index 2ce12dcac0f..7464d632f69 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -828,7 +828,7 @@ def _finalize_numpy(np, available): def _pyutilib_importer(): # On newer Pythons, PyUtilib import will fail, but only if a - # second-level module is imported. We will arbirtarily choose to + # second-level module is imported. We will arbitrarily choose to # check pyutilib.component (as that is the path exercised by the # pyomo.common.tempfiles deprecation path) import pyutilib From a64b96ec5e5720fd159a7a0f2277fc10fc45e92e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 13:00:19 -0700 Subject: [PATCH 6/9] Prevent resolution of pyutilib_available in environ import --- pyomo/dataportal/plugins/__init__.py | 6 +----- pyomo/dataportal/plugins/sheet.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyomo/dataportal/plugins/__init__.py b/pyomo/dataportal/plugins/__init__.py index e861233dc01..c3387af9d1e 100644 --- a/pyomo/dataportal/plugins/__init__.py +++ b/pyomo/dataportal/plugins/__init__.py @@ -9,8 +9,6 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from pyomo.common.dependencies import pyutilib, pyutilib_available - def load(): import pyomo.dataportal.plugins.csv_table @@ -19,6 +17,4 @@ def load(): import pyomo.dataportal.plugins.json_dict import pyomo.dataportal.plugins.text import pyomo.dataportal.plugins.xml_table - - if pyutilib_available: - import pyomo.dataportal.plugins.sheet + import pyomo.dataportal.plugins.sheet diff --git a/pyomo/dataportal/plugins/sheet.py b/pyomo/dataportal/plugins/sheet.py index bc7e4d06952..8672b9917da 100644 --- a/pyomo/dataportal/plugins/sheet.py +++ b/pyomo/dataportal/plugins/sheet.py @@ -18,9 +18,18 @@ # ) from pyomo.dataportal.factory import DataManagerFactory from pyomo.common.errors import ApplicationError -from pyomo.common.dependencies import attempt_import +from pyomo.common.dependencies import attempt_import, importlib, pyutilib -spreadsheet, spreadsheet_available = attempt_import('pyutilib.excel.spreadsheet') + +def _spreadsheet_importer(): + # verify pyutilib imported correctly the first time + pyutilib.component + return importlib.import_module('pyutilib.excel.spreadsheet') + + +spreadsheet, spreadsheet_available = attempt_import( + 'pyutilib.excel.spreadsheet', importer=_spreadsheet_importer +) def _attempt_open_excel(): From 8ad516c7e601fd0516bf860a2ec62a3ef8eed728 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 13:02:16 -0700 Subject: [PATCH 7/9] Clean up pyutilib import --- pyomo/common/dependencies.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index 7464d632f69..350762bc8ad 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -831,10 +831,8 @@ def _pyutilib_importer(): # second-level module is imported. We will arbitrarily choose to # check pyutilib.component (as that is the path exercised by the # pyomo.common.tempfiles deprecation path) - import pyutilib - import pyutilib.component - - return pyutilib + importlib.import_module('pyutilib.component') + return importlib.import_module('pyutilib') dill, dill_available = attempt_import('dill') From fd5b4daa672fab7bab4b90cd1bfdc095625f5c79 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 16:38:06 -0700 Subject: [PATCH 8/9] Remove distutils dependency in CI harnesses --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- .jenkins.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index f3f19b78591..ff24c731d94 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -592,8 +592,8 @@ jobs: echo "COVERAGE_PROCESS_START=$COVERAGE_RC" >> $GITHUB_ENV cp ${GITHUB_WORKSPACE}/.coveragerc ${COVERAGE_RC} echo "data_file=${COVERAGE_BASE}age" >> ${COVERAGE_RC} - SITE_PACKAGES=$($PYTHON_EXE -c "from distutils.sysconfig import \ - get_python_lib; print(get_python_lib())") + SITE_PACKAGES=$($PYTHON_EXE -c \ + "import sysconfig; print(sysconfig.get_path('purelib'))") echo "Python site-packages: $SITE_PACKAGES" echo 'import coverage; coverage.process_startup()' \ > ${SITE_PACKAGES}/run_coverage_at_startup.pth diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 13dc828c639..6e5604bea47 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -622,8 +622,8 @@ jobs: echo "COVERAGE_PROCESS_START=$COVERAGE_RC" >> $GITHUB_ENV cp ${GITHUB_WORKSPACE}/.coveragerc ${COVERAGE_RC} echo "data_file=${COVERAGE_BASE}age" >> ${COVERAGE_RC} - SITE_PACKAGES=$($PYTHON_EXE -c "from distutils.sysconfig import \ - get_python_lib; print(get_python_lib())") + SITE_PACKAGES=$($PYTHON_EXE -c \ + "import sysconfig; print(sysconfig.get_path('purelib'))") echo "Python site-packages: $SITE_PACKAGES" echo 'import coverage; coverage.process_startup()' \ > ${SITE_PACKAGES}/run_coverage_at_startup.pth diff --git a/.jenkins.sh b/.jenkins.sh index f31fef99377..544cb549175 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -77,7 +77,7 @@ if test -z "$MODE" -o "$MODE" == setup; then source python/bin/activate # Because modules set the PYTHONPATH, we need to make sure that the # virtualenv appears first - LOCAL_SITE_PACKAGES=`python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"` + LOCAL_SITE_PACKAGES=`python -c "import sysconfig; print(sysconfig.get_path('purelib'))"` export PYTHONPATH="$LOCAL_SITE_PACKAGES:$PYTHONPATH" # Set up Pyomo checkouts From e3df7bdaf62f5fadd4a76fb4dad9aa4df084ba2f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 7 Dec 2023 23:13:02 -0700 Subject: [PATCH 9/9] Remove distutils references --- pyomo/common/cmake_builder.py | 9 +++------ pyomo/contrib/appsi/build.py | 3 +-- pyomo/contrib/mcpp/build.py | 6 +++--- setup.py | 2 ++ 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pyomo/common/cmake_builder.py b/pyomo/common/cmake_builder.py index 71358c29fb2..bb612b43b72 100644 --- a/pyomo/common/cmake_builder.py +++ b/pyomo/common/cmake_builder.py @@ -32,11 +32,8 @@ def handleReadonly(function, path, excinfo): def build_cmake_project( targets, package_name=None, description=None, user_args=[], parallel=None ): - # Note: setuptools must be imported before distutils to avoid - # warnings / errors with recent setuptools distributions - from setuptools import Extension - import distutils.core - from distutils.command.build_ext import build_ext + from setuptools import Extension, Distribution + from setuptools.command.build_ext import build_ext class _CMakeBuild(build_ext, object): def run(self): @@ -122,7 +119,7 @@ def __init__(self, target_dir, user_args, parallel): 'ext_modules': ext_modules, 'cmdclass': {'build_ext': _CMakeBuild}, } - dist = distutils.core.Distribution(package_config) + dist = Distribution(package_config) basedir = os.path.abspath(os.path.curdir) try: tmpdir = os.path.abspath(tempfile.mkdtemp()) diff --git a/pyomo/contrib/appsi/build.py b/pyomo/contrib/appsi/build.py index 2a4e7bb785e..2c8d02dd3ac 100644 --- a/pyomo/contrib/appsi/build.py +++ b/pyomo/contrib/appsi/build.py @@ -63,8 +63,7 @@ def get_appsi_extension(in_setup=False, appsi_root=None): def build_appsi(args=[]): print('\n\n**** Building APPSI ****') - import setuptools - from distutils.dist import Distribution + from setuptools import Distribution from pybind11.setup_helpers import build_ext import pybind11.setup_helpers from pyomo.common.envvar import PYOMO_CONFIG_DIR diff --git a/pyomo/contrib/mcpp/build.py b/pyomo/contrib/mcpp/build.py index 95246e5278e..55c893335d2 100644 --- a/pyomo/contrib/mcpp/build.py +++ b/pyomo/contrib/mcpp/build.py @@ -64,8 +64,8 @@ def _generate_configuration(): def build_mcpp(): - import distutils.core - from distutils.command.build_ext import build_ext + from setuptools import Distribution + from setuptools.command.build_ext import build_ext class _BuildWithoutPlatformInfo(build_ext, object): # Python3.x puts platform information into the generated SO file @@ -87,7 +87,7 @@ def get_ext_filename(self, ext_name): print("\n**** Building MCPP library ****") package_config = _generate_configuration() package_config['cmdclass'] = {'build_ext': _BuildWithoutPlatformInfo} - dist = distutils.core.Distribution(package_config) + dist = Distribution(package_config) install_dir = os.path.join(envvar.PYOMO_CONFIG_DIR, 'lib') dist.get_command_obj('install_lib').install_dir = install_dir try: diff --git a/setup.py b/setup.py index b019abe91cb..dae62e72ca0 100644 --- a/setup.py +++ b/setup.py @@ -19,8 +19,10 @@ from setuptools import setup, find_packages, Command try: + # This works beginning in setuptools 40.7.0 (27 Jan 2019) from setuptools import DistutilsOptionError except ImportError: + # Needed for setuptools prior to 40.7.0 from distutils.errors import DistutilsOptionError