Skip to content

Commit

Permalink
Factor out method for setting variables for Python package installations
Browse files Browse the repository at this point in the history
Consolidates the multiple individual usages into a single function.
  • Loading branch information
Flamefire committed Sep 26, 2024
1 parent 6aa13e3 commit 7c6bfc9
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 50 deletions.
18 changes: 7 additions & 11 deletions easybuild/easyblocks/generic/pythonbundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,10 @@

from easybuild.easyblocks.generic.bundle import Bundle
from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd, set_py_env_vars
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import which
from easybuild.tools.modules import get_software_root
import easybuild.tools.environment as env


class PythonBundle(Bundle):
Expand Down Expand Up @@ -135,8 +134,7 @@ def prepare_step(self, *args, **kwargs):

def extensions_step(self, *args, **kwargs):
"""Install extensions (usually PythonPackages)"""
# don't add user site directory to sys.path (equivalent to python -s)
env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
set_py_env_vars(self.log)
super(PythonBundle, self).extensions_step(*args, **kwargs)

def test_step(self):
Expand Down Expand Up @@ -171,18 +169,16 @@ def make_module_extra(self, *args, **kwargs):
return txt

def load_module(self, *args, **kwargs):
"""(Re)set environment variables after loading module file.
Required here to ensure the variables are also defined for stand-alone installations,
because the environment is reset to the initial environment right before loading the module.
"""
Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software."""

super(PythonBundle, self).load_module(*args, **kwargs)

# Don't add user site directory to sys.path (equivalent to python -s),
# to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check.
# Required here to ensure that it is defined for sanity check commands of the bundle
# because the environment is reset to the initial environment right before loading the module
env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
# Set (again) in case the module changes it
env.setvar('PIP_REQUIRE_VIRTUALENV', 'false', verbose=False)
set_py_env_vars(self.log)

def sanity_check_step(self, *args, **kwargs):
"""Custom sanity check for bundle of Python package."""
Expand Down
42 changes: 12 additions & 30 deletions easybuild/easyblocks/generic/pythonpackage.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@

import easybuild.tools.environment as env
from easybuild.base import fancylogger
from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
Expand Down Expand Up @@ -189,8 +189,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None):
if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'):
# Python 3.12 removed distutils but has a core sysconfig module which is similar
pathname = 'platlib' if plat_specific else 'purelib'
vars = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars)
vars_param = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param)
else:
args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
Expand Down Expand Up @@ -438,15 +438,7 @@ def __init__(self, *args, **kwargs):
self.use_setup_py = False
self.determine_install_command()

# avoid that pip (ab)uses $HOME/.cache/pip
# cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
env.setvar('XDG_CACHE_HOME', os.path.join(self.builddir, 'xdg-cache-home'))
self.log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME'])
# Users or sites may require using a virtualenv for user installations
# We need to disable this to be able to install into the module
env.setvar('PIP_REQUIRE_VIRTUALENV', 'false')
# Don't let pip connect to PYPI to check for a new version
env.setvar('PIP_DISABLE_PIP_VERSION_CHECK', 'true')
set_py_env_vars(self.log)

def determine_install_command(self):
"""
Expand Down Expand Up @@ -734,9 +726,7 @@ def prepare_step(self, *args, **kwargs):
def configure_step(self):
"""Configure Python package build/install."""

# don't add user site directory to sys.path (equivalent to python -s)
# see https://www.python.org/dev/peps/pep-0370/
env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
set_py_env_vars(self.log)

if self.python_cmd is None:
self.prepare_python()
Expand Down Expand Up @@ -969,17 +959,14 @@ def run(self, *args, **kwargs):
step_method(self)()

def load_module(self, *args, **kwargs):
"""(Re)set environment variables after loading module file for this software.
Required here to ensure the variables are also defined for stand-alone installations,
because the environment is reset to the initial environment right before loading the module.
"""
Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software."""

super(PythonPackage, self).load_module(*args, **kwargs)

# don't add user site directory to sys.path (equivalent to python -s),
# to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check;
# required here to ensure that it is defined for stand-alone installations,
# because the environment is reset to the initial environment right before loading the module
env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
env.setvar('PIP_REQUIRE_VIRTUALENV', 'false', verbose=False)
set_py_env_vars(self.log)

def sanity_check_step(self, *args, **kwargs):
"""
Expand All @@ -996,14 +983,9 @@ def sanity_check_step(self, *args, **kwargs):
extra_modules = kwargs.get('extra_modules', None)
self.fake_mod_data = self.sanity_check_load_module(extension=extension, extra_modules=extra_modules)

# don't add user site directory to sys.path (equivalent to python -s)
# see https://www.python.org/dev/peps/pep-0370/;
# must be set here to ensure that it is defined when running sanity check for extensions,
# since load_module is not called for every extension,
# to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check;
# Must be called here since load_module is not called for every extension,
# see also https://github.com/easybuilders/easybuild-easyblocks/issues/1877
env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
env.setvar('PIP_REQUIRE_VIRTUALENV', 'false', verbose=False)
set_py_env_vars(self.log)

if self.cfg.get('download_dep_fail', False):
self.log.info("Detection of downloaded dependencies enabled, checking output of installation command...")
Expand Down
37 changes: 31 additions & 6 deletions easybuild/easyblocks/p/python.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,17 @@

EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'

# Environment variables and values to avoid common issues during Python package installations and usage in EasyBuild
PY_ENV_VARS = {
# don't add user site directory to sys.path (equivalent to python -s), see https://www.python.org/dev/peps/pep-0370
'PYTHONNOUSERSITE': '1',
# Users or sites may require using a virtualenv for user installations
# We need to disable this to be able to install into modules
'PIP_REQUIRE_VIRTUALENV': 'false',
# Don't let pip connect to PYPI to check for a new version
'PIP_DISABLE_PIP_VERSION_CHECK': 'true',
}

# We want the following import order:
# 1. Packages installed into VirtualEnv
# 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules)
Expand Down Expand Up @@ -109,6 +120,22 @@
""" % {'EBPYTHONPREFIXES': EBPYTHONPREFIXES}


def set_py_env_vars(log, verbose=False):
"""Set environment variables required/useful for installing or using Python packages"""

py_vars = PY_ENV_VARS.copy()
# avoid that pip (ab)uses $HOME/.cache/pip
# cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
py_vars['XDG_CACHE_HOME'] = os.path.join(tempfile.gettempdir(), 'xdg-cache-home')
# Only set (all) environment variables if any has a different value to
# avoid (non)changes (and log messages) for each package in a bundle
set_required = any(os.environ.get(name, None) != value for name, value in py_vars.items())
if set_required:
for name, value in py_vars.items():
env.setvar(name, value, verbose=verbose)
log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME'])


class EB_Python(ConfigureMake):
"""Support for building/installing Python
- default configure/build_step/make install works fine
Expand Down Expand Up @@ -161,9 +188,9 @@ def __init__(self, *args, **kwargs):
}

exts_default_options = self.cfg.get_ref('exts_default_options')
for key in ext_defaults:
for key, value in ext_defaults.items():
if key not in exts_default_options:
exts_default_options[key] = ext_defaults[key]
exts_default_options[key] = value
self.log.debug("exts_default_options: %s", self.cfg['exts_default_options'])

self.install_pip = self.cfg['install_pip']
Expand Down Expand Up @@ -296,8 +323,7 @@ def prepare_for_extensions(self):
self.cfg['exts_defaultclass'] = "PythonPackage"
self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES

# don't add user site directory to sys.path (equivalent to python -s)
env.setvar('PYTHONNOUSERSITE', '1')
set_py_env_vars(self.log)

# don't pass down any build/install options that may have been specified
# 'make' options do not make sense for when building/installing Python libraries (usually via 'python setup.py')
Expand Down Expand Up @@ -434,9 +460,8 @@ def configure_step(self):
env.setvar('TCLTK_CFLAGS', '-I%s/include -I%s/include' % (tcl, tk))
env.setvar('TCLTK_LIBS', tcltk_libs)

# don't add user site directory to sys.path (equivalent to python -s)
# This matters e.g. when python installs the bundled pip & setuptools (for >= 3.4)
env.setvar('PYTHONNOUSERSITE', '1')
set_py_env_vars(self.log)

super(EB_Python, self).configure_step()

Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/t/tensorflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version
from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES, PY_ENV_VARS
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools import run, LooseVersion
from easybuild.tools.build_log import EasyBuildError, print_warning
Expand Down Expand Up @@ -894,7 +894,7 @@ def build_step(self):
action_env['EBPYTHONPREFIXES'] = INHERIT

# Ignore user environment for Python
action_env['PYTHONNOUSERSITE'] = '1'
action_env.update(PY_ENV_VARS)

# TF 2 (final) sets this in configure
if LooseVersion(self.version) < LooseVersion('2.0'):
Expand Down
3 changes: 2 additions & 1 deletion easybuild/easyblocks/t/tensorflow_compression.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import easybuild.tools.environment as env
from easybuild.easyblocks.generic.pythonpackage import PythonPackage
from easybuild.easyblocks.python import PY_ENV_VARS
from easybuild.tools import LooseVersion
from easybuild.tools.modules import get_software_version
from easybuild.tools.run import run_cmd
Expand Down Expand Up @@ -104,7 +105,7 @@ def build_step(self):
action_env['EBPYTHONPREFIXES'] = INHERIT

# Ignore user environment for Python
action_env['PYTHONNOUSERSITE'] = '1'
action_env.update(PY_ENV_VARS)

# Use the same configuration (i.e. environment) for compiling and using host tools
# This means that our action_envs are (almost) always passed
Expand Down

0 comments on commit 7c6bfc9

Please sign in to comment.