diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 636ad602..0779431c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,10 +24,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' @@ -64,10 +64,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3261d728..5ecdb2fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,6 @@ jobs: # versions by django-environ will continue for as long as possible, # and may be discontinued at any time. include: - - python: '3.5' - os: ubuntu-20.04 - python: '3.6' os: ubuntu-20.04 - python: '3.7' @@ -68,12 +66,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 with: fetch-depth: 5 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.7.0 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4cf741a6..07dd2008 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 795a1533..fc596beb 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,10 +25,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e079bee6..e03e0d8b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' diff --git a/BACKERS.rst b/BACKERS.rst index 4ac3b73c..dc014446 100644 --- a/BACKERS.rst +++ b/BACKERS.rst @@ -21,7 +21,7 @@ Thank you to all our backers! |ocbackerimage| .. |ocsponsor0| image:: https://opencollective.com/django-environ/sponsor/0/avatar.svg - :target: https://triplebyte.com/ + :target: https://opencollective.com/triplebyte :alt: Sponsor .. |ocsponsor1| image:: https://images.opencollective.com/static/images/become_sponsor.svg :target: https://opencollective.com/django-environ/contribute/sponsors-3474/checkout diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b310499..78b6a0bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,10 +5,41 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.10.0`_ - 2-March-2023 +`v0.11.0`_ - 30-August-2023 ------------------------------- Added +++++ +- Added support for Django 4.2 + `#456 `_. +- Added support for secure Elasticsearch connections + `#463 `_. +- Added variable expansion + `#468 `_. +- Added capability to handle comments after #, after quoted values, + like ``KEY= 'part1 # part2' # comment`` + `#475 `_. +- Added support for ``interpolate`` parameter + `#415 `_. + +Changed ++++++++ +- Used ``mssql-django`` as engine for SQL Server + `#446 `_. +- Changed handling bool values, stripping whitespace around value + `#475 `_. +- Use ``importlib.util.find_spec`` to ``replace pkgutil.find_loader`` + `#482 `_. + + +Removed ++++++++ +- Removed support of Python 3.5. + + +`v0.10.0`_ - 2-March-2023 +------------------------- +Added ++++++ - Use the core redis library by default if running Django >= 4.0 `#356 `_. - Value of dict can now contain an equal sign @@ -29,7 +60,7 @@ Deprecated Changed +++++++ - Used UTF-8 as a encoding when open ``.env`` file. -- Provided access to ```DB_SCHEMES`` through ``cls`` rather than +- Provided access to ``DB_SCHEMES`` through ``cls`` rather than ``Env`` in ``db_url_config`` `#414 `_. - Correct CI workflow to use supported Python versions/OS matrix @@ -341,7 +372,8 @@ Added - Initial release. -.. _v0.10.0: https://github.com/joke2k/django-environ/compare/v0.9.0...develop +.. _v0.11.0: https://github.com/joke2k/django-environ/compare/v0.10.0...develop +.. _v0.10.0: https://github.com/joke2k/django-environ/compare/v0.9.0...v0.10.0 .. _v0.9.0: https://github.com/joke2k/django-environ/compare/v0.8.1...v0.9.0 .. _v0.8.1: https://github.com/joke2k/django-environ/compare/v0.8.0...v0.8.1 .. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 29d555ef..549b7f84 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -74,5 +74,5 @@ Resources --------- * `How to Contribute to Open Source `_ -* `Using Pull Requests `_ -* `Writing good commit messages `_ +* `Using Pull Requests `_ +* `Writing good commit messages `_ diff --git a/README.rst b/README.rst index cf999e23..528d1df1 100644 --- a/README.rst +++ b/README.rst @@ -126,8 +126,8 @@ its documentation lives at `Read the Docs `_, and the latest release on `PyPI `_. -It’s rigorously tested on Python 3.5+, and officially supports -Django 1.11, 2.2, 3., 3.1, 3.2, 4.0 and 4.1. +It’s rigorously tested on Python 3.6+, and officially supports +Django 1.11, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1 and 4.2. If you'd like to contribute to ``django-environ`` you're most welcome! diff --git a/docs/conf.py b/docs/conf.py index 765f9947..8beac1f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021-2022, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -12,12 +12,10 @@ import codecs import os -import sys import re - +import sys from datetime import date - PROJECT_DIR = os.path.abspath('..') sys.path.insert(0, PROJECT_DIR) @@ -71,7 +69,7 @@ def find_version(meta_file): # The suffix of source filenames. source_suffix = ".rst" -# Allow non-local URIs so we can have images in CHANGELOG etc. +# Allow non-local URIs, so we can have images in CHANGELOG etc. suppress_warnings = [ "image.nonlocal_uri", ] diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 62473838..2ab100fd 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -23,6 +23,28 @@ And use it with ``settings.py`` as follows: :start-after: -code-begin- :end-before: -overview- +Variables can contain references to another variables: ``$VAR`` or ``${VAR}``. +Referenced variables are searched in the environment and within all definitions +in the ``.env`` file. References are checked for recursion (self-reference). +Exception is thrown if any reference results in infinite loop on any level +of recursion. Variable values are substituted similar to shell parameter +expansion. Example: + +.. code-block:: shell + + # shell + export POSTGRES_USERNAME='user' POSTGRES_PASSWORD='SECRET' + +.. code-block:: shell + + # .env + POSTGRES_HOSTNAME='example.com' + POSTGRES_DB='database' + DATABASE_URL="postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOSTNAME}:5432/${POSTGRES_DB}" + +The value of ``DATABASE_URL`` variable will become +``postgres://user:SECRET@example.com:5432/database``. + The ``.env`` file should be specific to the environment and not checked into version control, it is best practice documenting the ``.env`` file with an example. For example, you can also add ``.env.dist`` with a template of your variables to diff --git a/docs/tips.rst b/docs/tips.rst index 1915fac9..ab59f691 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -226,7 +226,7 @@ Proxy value =========== Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to -``environ.Env()`` to enable this feature: +``environ.Env()`` to enable this feature (``True`` by default): .. code-block:: python @@ -236,7 +236,7 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to # BAR=FOO # PROXY=$BAR - >>> print env.str('PROXY') + >>> print(env.str('PROXY')) FOO diff --git a/docs/types.rst b/docs/types.rst index 3fcdcbbd..c099bf91 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -156,10 +156,10 @@ For more detailed example see ":ref:`complex_dict_format`". :py:meth:`~.environ.Env.search_url` supports the following URL schemas: -* Elasticsearch: ``elasticsearch://`` -* Elasticsearch2: ``elasticsearch2://`` -* Elasticsearch5: ``elasticsearch5://`` -* Elasticsearch7: ``elasticsearch7://`` +* Elasticsearch: ``elasticsearch://`` (http) or ``elasticsearchs://`` (https) +* Elasticsearch2: ``elasticsearch2://`` (http) or ``elasticsearch2s://`` (https) +* Elasticsearch5: ``elasticsearch5://`` (http) or ``elasticsearch5s://`` (https) +* Elasticsearch7: ``elasticsearch7://`` (http) or ``elasticsearch7s://`` (https) * Solr: ``solr://`` * Whoosh: ``whoosh://`` * Xapian: ``xapian://`` diff --git a/environ/__init__.py b/environ/__init__.py index c79736cc..54e9465c 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021-2022, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -21,7 +21,7 @@ __copyright__ = 'Copyright (C) 2013-2022 Daniele Faraglia' """The copyright notice of the package.""" -__version__ = '0.10.0' +__version__ = '0.11.0' """The version of the package.""" __license__ = 'MIT' diff --git a/environ/compat.py b/environ/compat.py index 0cec1e5e..49b5b480 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -8,15 +8,14 @@ """This module handles import compatibility issues.""" -from pkgutil import find_loader +from importlib.util import find_spec - -if find_loader('simplejson'): +if find_spec('simplejson'): import simplejson as json else: import json -if find_loader('django'): +if find_spec('django'): from django import VERSION as DJANGO_VERSION from django.core.exceptions import ImproperlyConfigured else: @@ -28,14 +27,17 @@ class ImproperlyConfigured(Exception): def choose_rediscache_driver(): """Backward compatibility for RedisCache driver.""" + + # django-redis library takes precedence + if find_spec('django_redis'): + return 'django_redis.cache.RedisCache' + # use built-in support if Django 4+ if DJANGO_VERSION is not None and DJANGO_VERSION >= (4, 0): return 'django.core.cache.backends.redis.RedisCache' # back compatibility with redis_cache package - if find_loader('redis_cache'): - return 'redis_cache.RedisCache' - return 'django_redis.cache.RedisCache' + return 'redis_cache.RedisCache' def choose_postgres_driver(): @@ -49,7 +51,7 @@ def choose_postgres_driver(): def choose_pymemcache_driver(): """Backward compatibility for pymemcache.""" old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2) - if old_django or not find_loader('pymemcache'): + if old_django or not find_spec('pymemcache'): # The original backend choice for the 'pymemcache' scheme is # unfortunately 'pylibmc'. return 'django.core.cache.backends.memcached.PyLibMCCache' diff --git a/environ/environ.py b/environ/environ.py index 9ca00a8e..f35470c5 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -17,10 +17,13 @@ import os import re import sys +import threading import warnings +from os.path import expandvars from urllib.parse import ( parse_qs, ParseResult, + quote, unquote, unquote_plus, urlparse, @@ -36,15 +39,12 @@ ) from .fileaware_mapping import FileAwareMapping -try: - from os import PathLike -except ImportError: # Python 3.5 support - from pathlib import PurePath as PathLike - -Openable = (str, PathLike) - +Openable = (str, os.PathLike) logger = logging.getLogger(__name__) +# Variables which values should not be expanded +NOT_EXPANDED = 'DJANGO_SECRET_KEY', 'CACHE_URL' + def _cast(value): # Safely evaluate an expression node or a string containing a Python @@ -65,11 +65,15 @@ def _cast_urlstr(v): return unquote(v) if isinstance(v, str) else v +def _urlparse_quote(url): + return urlparse(quote(url, safe=':/?&=@')) + + class NoValue: """Represent of no value object.""" def __repr__(self): - return '<{}>'.format(self.__class__.__name__) + return f'<{self.__class__.__name__}>' class Env: @@ -108,7 +112,6 @@ class Env: URL_CLASS = ParseResult POSTGRES_FAMILY = ['postgres', 'postgresql', 'psql', 'pgsql', 'postgis'] - ELASTICSEARCH_FAMILY = ['elasticsearch' + x for x in ['', '2', '5', '7']] DEFAULT_DATABASE_ENV = 'DATABASE_URL' DB_SCHEMES = { @@ -121,7 +124,7 @@ class Env: 'mysql2': 'django.db.backends.mysql', 'mysql-connector': 'mysql.connector.django', 'mysqlgis': 'django.contrib.gis.db.backends.mysql', - 'mssql': 'sql_server.pyodbc', + 'mssql': 'mssql', 'oracle': 'django.db.backends.oracle', 'pyodbc': 'sql_server.pyodbc', 'redshift': 'django_redshift_backend', @@ -186,12 +189,20 @@ class Env: "xapian": "haystack.backends.xapian_backend.XapianEngine", "simple": "haystack.backends.simple_backend.SimpleEngine", } + ELASTICSEARCH_FAMILY = [scheme + s for scheme in SEARCH_SCHEMES + if scheme.startswith("elasticsearch") + for s in ('', 's')] CLOUDSQL = 'cloudsql' - def __init__(self, **scheme): + VAR = re.compile(r'(?[A-Z_][0-9A-Z_]*)}?', + re.IGNORECASE) + + def __init__(self, interpolate=True, **scheme): + self._local = threading.local() self.smart_cast = True self.escape_proxy = False self.prefix = "" + self.interpolate = interpolate self.scheme = scheme def __call__(self, var, cast=None, default=NOTSET, parse_default=False): @@ -342,9 +353,13 @@ def path(self, var, default=NOTSET, **kwargs): """ return Path(self.get_value(var, default=default), **kwargs) - def get_value(self, var, cast=None, default=NOTSET, parse_default=False): + def get_value(self, var, cast=None, # pylint: disable=R0913 + default=NOTSET, parse_default=False, add_prefix=True): """Return value for given environment variable. + - Expand variables referenced as ``$VAR`` or ``${VAR}``. + - Detect infinite recursion in expansion (self-reference). + :param str var: Name of variable. :param collections.abc.Callable or None cast: @@ -353,15 +368,33 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): If var not present in environ, return this instead. :param bool parse_default: Force to parse default. + :param bool add_prefix: + Whether to add prefix to variable name. :returns: Value from environment or default (if set). :rtype: typing.IO[typing.Any] """ - + var_name = f'{self.prefix}{var}' if add_prefix else var + if not hasattr(self._local, 'vars'): + self._local.vars = set() + if var_name in self._local.vars: + error_msg = f"Environment variable '{var_name}' recursively "\ + "references itself (eventually)" + raise ImproperlyConfigured(error_msg) + + self._local.vars.add(var_name) + try: + return self._get_value( + var_name, cast=cast, default=default, + parse_default=parse_default) + finally: + self._local.vars.remove(var_name) + + def _get_value(self, var_name, cast=None, default=NOTSET, + parse_default=False): logger.debug( "get '%s' casted as '%s' with default '%s'", - var, cast, default) + var_name, cast, default) - var_name = "{}{}".format(self.prefix, var) if var_name in self.scheme: var_info = self.scheme[var_name] @@ -387,26 +420,38 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): value = self.ENVIRON[var_name] except KeyError as exc: if default is self.NOTSET: - error_msg = "Set the {} environment variable".format(var) + error_msg = f'Set the {var_name} environment variable' raise ImproperlyConfigured(error_msg) from exc value = default + # Expand variables + if self.interpolate and isinstance(value, (bytes, str)) \ + and var_name not in NOT_EXPANDED: + def repl(match_): + return self.get_value( + match_.group('name'), cast=cast, default=default, + parse_default=parse_default, add_prefix=False) + + is_bytes = isinstance(value, bytes) + if is_bytes: + value = value.decode('utf-8') + value = self.VAR.sub(repl, value) + value = expandvars(value) + if is_bytes: + value = value.encode('utf-8') + # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' escape = rb'\$' if isinstance(value, bytes) else r'\$' - if hasattr(value, 'startswith') and value.startswith(prefix): - value = value.lstrip(prefix) - value = self.get_value(value, cast=cast, default=default) if self.escape_proxy and hasattr(value, 'replace'): value = value.replace(escape, prefix) # Smart casting - if self.smart_cast: - if cast is None and default is not None and \ - not isinstance(default, NoValue): - cast = type(default) + if self.smart_cast and cast is None and default is not None \ + and not isinstance(default, NoValue): + cast = type(default) value = None if default is None and value == '' else value @@ -430,7 +475,7 @@ def parse_value(cls, value, cast): try: value = int(value) != 0 except ValueError: - value = value.lower() in cls.BOOLEAN_TRUE_STRINGS + value = value.lower().strip() in cls.BOOLEAN_TRUE_STRINGS elif isinstance(cast, list): value = list(map(cast[0], [x for x in value.split(',') if x])) elif isinstance(cast, tuple): @@ -467,14 +512,16 @@ def parse_value(cls, value, cast): if len(parts) == 1: float_str = parts[0] else: - float_str = "{}.{}".format(''.join(parts[0:-1]), parts[-1]) + float_str = f"{''.join(parts[0:-1])}.{parts[-1]}" value = float(float_str) else: value = cast(value) return value @classmethod + # pylint: disable=too-many-statements def db_url_config(cls, url, engine=None): + # pylint: enable-msg=too-many-statements """Parse an arbitrary database URL. Supports the following URL schemas: @@ -509,10 +556,17 @@ def db_url_config(cls, url, engine=None): 'NAME': ':memory:' } # note: no other settings are required for sqlite - url = urlparse(url) + try: + url = urlparse(url) + # handle Invalid IPv6 URL + except ValueError: + url = _urlparse_quote(url) config = {} + # handle unexpected URL schemes with special characters + if not url.path: + url = _urlparse_quote(urlunparse(url)) # Remove query strings. path = url.path[1:] path = unquote_plus(path.split('?', 2)[0]) @@ -524,15 +578,15 @@ def db_url_config(cls, url, engine=None): # sqlalchemy) path = ':memory:' if url.netloc: - warnings.warn('SQLite URL contains host component %r, ' - 'it will be ignored' % url.netloc, stacklevel=3) + warnings.warn( + f'SQLite URL contains host component {url.netloc!r}, ' + 'it will be ignored', + stacklevel=3 + ) if url.scheme == 'ldap': - path = '{scheme}://{hostname}'.format( - scheme=url.scheme, - hostname=url.hostname, - ) + path = f'{url.scheme}://{url.hostname}' if url.port: - path += ':{port}'.format(port=url.port) + path += f':{url.port}' user_host = url.netloc.rsplit('@', 1) if url.scheme in cls.POSTGRES_FAMILY and ',' in user_host[-1]: @@ -595,7 +649,7 @@ def db_url_config(cls, url, engine=None): config['ENGINE'] = cls.DB_SCHEMES[config['ENGINE']] if not config.get('ENGINE', False): - warnings.warn("Engine not recognized from url: {}".format(config)) + warnings.warn(f'Engine not recognized from url: {config}') return {} return config @@ -617,9 +671,7 @@ def cache_url_config(cls, url, backend=None): url = urlparse(url) if url.scheme not in cls.CACHE_SCHEMES: - raise ImproperlyConfigured( - 'Invalid cache schema {}'.format(url.scheme) - ) + raise ImproperlyConfigured(f'Invalid cache schema {url.scheme}') location = url.netloc.split(',') if len(location) == 1: @@ -707,7 +759,7 @@ def email_url_config(cls, url, backend=None): if backend: config['EMAIL_BACKEND'] = backend elif url.scheme not in cls.EMAIL_SCHEMES: - raise ImproperlyConfigured('Invalid email schema %s' % url.scheme) + raise ImproperlyConfigured(f'Invalid email schema {url.scheme}') elif url.scheme in cls.EMAIL_SCHEMES: config['EMAIL_BACKEND'] = cls.EMAIL_SCHEMES[url.scheme] @@ -728,6 +780,72 @@ def email_url_config(cls, url, backend=None): return config + @classmethod + def _parse_common_search_params(cls, url): + cfg = {} + prs = {} + + if not url.query or str(url.query) == '': + return cfg, prs + + prs = parse_qs(url.query) + if 'EXCLUDED_INDEXES' in prs: + cfg['EXCLUDED_INDEXES'] = prs['EXCLUDED_INDEXES'][0].split(',') + if 'INCLUDE_SPELLING' in prs: + val = prs['INCLUDE_SPELLING'][0] + cfg['INCLUDE_SPELLING'] = cls.parse_value(val, bool) + if 'BATCH_SIZE' in prs: + cfg['BATCH_SIZE'] = cls.parse_value(prs['BATCH_SIZE'][0], int) + return cfg, prs + + @classmethod + def _parse_elasticsearch_search_params(cls, url, path, secure, params): + cfg = {} + split = path.rsplit('/', 1) + + if len(split) > 1: + path = '/'.join(split[:-1]) + index = split[-1] + else: + path = "" + index = split[0] + + cfg['URL'] = urlunparse( + ('https' if secure else 'http', url[1], path, '', '', '') + ) + if 'TIMEOUT' in params: + cfg['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + if 'KWARGS' in params: + cfg['KWARGS'] = params['KWARGS'][0] + cfg['INDEX_NAME'] = index + return cfg + + @classmethod + def _parse_solr_search_params(cls, url, path, params): + cfg = {} + cfg['URL'] = urlunparse(('http',) + url[1:2] + (path,) + ('', '', '')) + if 'TIMEOUT' in params: + cfg['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + if 'KWARGS' in params: + cfg['KWARGS'] = params['KWARGS'][0] + return cfg + + @classmethod + def _parse_whoosh_search_params(cls, params): + cfg = {} + if 'STORAGE' in params: + cfg['STORAGE'] = params['STORAGE'][0] + if 'POST_LIMIT' in params: + cfg['POST_LIMIT'] = cls.parse_value(params['POST_LIMIT'][0], int) + return cfg + + @classmethod + def _parse_xapian_search_params(cls, params): + cfg = {} + if 'FLAGS' in params: + cfg['FLAGS'] = params['FLAGS'][0] + return cfg + @classmethod def search_url_config(cls, url, engine=None): """Parse an arbitrary search URL. @@ -739,88 +857,48 @@ def search_url_config(cls, url, engine=None): :return: Parsed search URL. :rtype: dict """ - config = {} - url = urlparse(url) if not isinstance(url, cls.URL_CLASS) else url # Remove query strings. - path = url.path[1:] - path = unquote_plus(path.split('?', 2)[0]) - - if url.scheme not in cls.SEARCH_SCHEMES: - raise ImproperlyConfigured( - 'Invalid search schema %s' % url.scheme - ) - config["ENGINE"] = cls.SEARCH_SCHEMES[url.scheme] + path = unquote_plus(url.path[1:].split('?', 2)[0]) + + scheme = url.scheme + secure = False + # elasticsearch supports secure schemes, similar to http -> https + if scheme in cls.ELASTICSEARCH_FAMILY and scheme.endswith('s'): + scheme = scheme[:-1] + secure = True + if scheme not in cls.SEARCH_SCHEMES: + raise ImproperlyConfigured(f'Invalid search schema {url.scheme}') + config['ENGINE'] = cls.SEARCH_SCHEMES[scheme] # check commons params - params = {} # type: dict - if url.query: - params = parse_qs(url.query) - if 'EXCLUDED_INDEXES' in params: - config['EXCLUDED_INDEXES'] \ - = params['EXCLUDED_INDEXES'][0].split(',') - if 'INCLUDE_SPELLING' in params: - config['INCLUDE_SPELLING'] = cls.parse_value( - params['INCLUDE_SPELLING'][0], - bool - ) - if 'BATCH_SIZE' in params: - config['BATCH_SIZE'] = cls.parse_value( - params['BATCH_SIZE'][0], - int - ) + cfg, params = cls._parse_common_search_params(url) + config.update(cfg) if url.scheme == 'simple': return config - if url.scheme in ['solr'] + cls.ELASTICSEARCH_FAMILY: - if 'KWARGS' in params: - config['KWARGS'] = params['KWARGS'][0] # remove trailing slash - if path.endswith("/"): + if path.endswith('/'): path = path[:-1] if url.scheme == 'solr': - config['URL'] = urlunparse( - ('http',) + url[1:2] + (path,) + ('', '', '') - ) - if 'TIMEOUT' in params: - config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + config.update(cls._parse_solr_search_params(url, path, params)) return config if url.scheme in cls.ELASTICSEARCH_FAMILY: - split = path.rsplit("/", 1) - - if len(split) > 1: - path = "/".join(split[:-1]) - index = split[-1] - else: - path = "" - index = split[0] - - config['URL'] = urlunparse( - ('http',) + url[1:2] + (path,) + ('', '', '') - ) - if 'TIMEOUT' in params: - config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) - config['INDEX_NAME'] = index + config.update(cls._parse_elasticsearch_search_params( + url, path, secure, params)) return config config['PATH'] = '/' + path if url.scheme == 'whoosh': - if 'STORAGE' in params: - config['STORAGE'] = params['STORAGE'][0] - if 'POST_LIMIT' in params: - config['POST_LIMIT'] = cls.parse_value( - params['POST_LIMIT'][0], - int - ) + config.update(cls._parse_whoosh_search_params(params)) elif url.scheme == 'xapian': - if 'FLAGS' in params: - config['FLAGS'] = params['FLAGS'][0] + config.update(cls._parse_xapian_search_params(params)) if engine: config['ENGINE'] = engine @@ -894,9 +972,17 @@ def _keep_escaped_format_characters(match): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: key, val = m1.group(1), m1.group(2) - m2 = re.match(r"\A'(.*)'\Z", val) + # Look for value in quotes, ignore post-# comments + # (outside quotes) + m2 = re.match(r"\A\s*'(?".format(self.__root__) + return f'' def __str__(self): return self.__root__ @@ -1050,5 +1136,6 @@ def _absolute_join(base, *paths, **kwargs): absolute_path = os.path.abspath(os.path.join(base, *paths)) if kwargs.get('required', False) and not os.path.exists(absolute_path): raise ImproperlyConfigured( - "Create required path: {}".format(absolute_path)) + f'Create required path: {absolute_path}' + ) return absolute_path diff --git a/setup.py b/setup.py index d317bc0a..30525752 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # # This file is part of the django-environ. # -# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -10,19 +10,21 @@ import codecs import re -import sys -import warnings from os import path from setuptools import find_packages, setup - -if sys.version_info < (3, 6): - warnings.warn( - "Support of Python < 3.6 is deprecated" - "and will be removed in a future release.", - DeprecationWarning - ) +# Use this code block for future deprecations of Python version: +# +# import warnings +# import sys +# +# if sys.version_info < (3, 6): +# warnings.warn( +# "Support of Python < 3.6 is deprecated" +# "and will be removed in a future release.", +# DeprecationWarning +# ) def read_file(filepath): @@ -120,7 +122,7 @@ def get_version_string(): return version_string -# What does this project relate to. +# What does this project relate to? KEYWORDS = [ 'environment', 'django', @@ -142,6 +144,7 @@ def get_version_string(): 'Framework :: Django :: 3.2', 'Framework :: Django :: 4.0', 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', 'Operating System :: OS Independent', @@ -150,7 +153,6 @@ def get_version_string(): 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -228,7 +230,7 @@ def get_version_string(): platforms=['any'], include_package_data=True, zip_safe=False, - python_requires='>=3.5,<4', + python_requires='>=3.6,<4', install_requires=INSTALL_REQUIRES, dependency_links=DEPENDENCY_LINKS, extras_require=EXTRAS_REQUIRE, diff --git a/tests/fixtures.py b/tests/fixtures.py index 6990d9af..69e5e90f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -38,6 +38,8 @@ class FakeEnv: @classmethod def generate_data(cls): return dict(STR_VAR='bar', + STR_QUOTED_IGNORE_COMMENT='foo', + STR_QUOTED_INCLUDE_HASH='foo # with hash', MULTILINE_STR_VAR='foo\\nbar', MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---', MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---', @@ -50,12 +52,14 @@ def generate_data(cls): BOOL_TRUE_STRING_LIKE_INT='1', BOOL_TRUE_INT=1, BOOL_TRUE_STRING_LIKE_BOOL='True', + BOOL_TRUE_STRING_LIKE_BOOL_WITH_COMMENT='True', BOOL_TRUE_STRING_1='on', BOOL_TRUE_STRING_2='ok', BOOL_TRUE_STRING_3='yes', BOOL_TRUE_STRING_4='y', BOOL_TRUE_STRING_5='true', BOOL_TRUE_BOOL=True, + BOOL_TRUE_BOOL_WITH_COMMENT=True, BOOL_FALSE_STRING_LIKE_INT='0', BOOL_FALSE_INT=0, BOOL_FALSE_STRING_LIKE_BOOL='False', @@ -65,7 +69,8 @@ def generate_data(cls): INT_LIST='42,33', INT_TUPLE='(42,33)', MIX_TUPLE='(42,Test)', - STR_LIST_WITH_SPACES=' foo, bar', + STR_LIST_WITH_SPACES=' foo, spaces', + STR_LIST_WITH_SPACES_QUOTED="' foo', ' quoted'", EMPTY_LIST='', DICT_VAR='foo=bar,test=on', DICT_WITH_EQ_VAR='key1=sub_key1=sub_value1,key2=value2', diff --git a/tests/test_cache.py b/tests/test_cache.py index a762c1b1..a8aff161 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -107,8 +107,8 @@ def test_pymemcache_compat(django_version, pymemcache_installed): old = 'django.core.cache.backends.memcached.PyLibMCCache' new = 'django.core.cache.backends.memcached.PyMemcacheCache' with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): - with mock.patch('environ.compat.find_loader') as mock_find_loader: - mock_find_loader.return_value = pymemcache_installed + with mock.patch('environ.compat.find_spec') as mock_find_spec: + mock_find_spec.return_value = pymemcache_installed driver = environ.compat.choose_pymemcache_driver() if django_version and django_version < (3, 2): assert driver == old @@ -117,21 +117,22 @@ def test_pymemcache_compat(django_version, pymemcache_installed): @pytest.mark.parametrize('django_version', ((4, 0), (3, 2), None)) -@pytest.mark.parametrize('redis_cache_installed', (True, False)) -def test_rediscache_compat(django_version, redis_cache_installed): +@pytest.mark.parametrize('django_redis_installed', (True, False)) +def test_rediscache_compat(django_version, django_redis_installed): django_new = 'django.core.cache.backends.redis.RedisCache' redis_cache = 'redis_cache.RedisCache' - django_old = 'django_redis.cache.RedisCache' + django_redis = 'django_redis.cache.RedisCache' with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): - with mock.patch('environ.compat.find_loader') as mock_find_loader: - mock_find_loader.return_value = redis_cache_installed + with mock.patch('environ.compat.find_spec') as mock_find_spec: + mock_find_spec.return_value = django_redis_installed driver = environ.compat.choose_rediscache_driver() - if django_version and django_version >= (4, 0): + if django_redis_installed: + assert driver == django_redis + elif django_version and django_version >= (4, 0): assert driver == django_new else: - assert driver == redis_cache if redis_cache_installed else django_old - + assert driver == redis_cache def test_redis_parsing(): url = ('rediscache://127.0.0.1:6379/1?client_class=' diff --git a/tests/test_db.py b/tests/test_db.py index c4074131..8101a4a7 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -118,7 +118,7 @@ # mysql://user:password@host/dbname ('mssql://enigma:secret@example.com/dbname' '?driver=ODBC Driver 13 for SQL Server', - 'sql_server.pyodbc', + 'mssql', 'dbname', 'example.com', 'enigma', @@ -127,12 +127,28 @@ # mysql://user:password@host:port/dbname ('mssql://enigma:secret@amazonaws.com\\insnsnss:12345/dbname' '?driver=ODBC Driver 13 for SQL Server', - 'sql_server.pyodbc', + 'mssql', 'dbname', 'amazonaws.com\\insnsnss', 'enigma', 'secret', 12345), + # mysql://user:password@host:port/dbname + ('mysql://enigma:><{~!@#$%^&*}[]@example.com:1234/dbname', + 'django.db.backends.mysql', + 'dbname', + 'example.com', + 'enigma', + '><{~!@#$%^&*}[]', + 1234), + # mysql://user:password@host/dbname + ('mysql://enigma:]password]@example.com/dbname', + 'django.db.backends.mysql', + 'dbname', + 'example.com', + 'enigma', + ']password]', + ''), ], ids=[ 'postgres', @@ -149,6 +165,8 @@ 'ldap', 'mssql', 'mssql_port', + 'mysql_password_special_chars', + 'mysql_invalid_ipv6_password', ], ) def test_db_parsing(url, engine, name, host, user, passwd, port): diff --git a/tests/test_env.py b/tests/test_env.py index 396c2123..1f2df156 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -112,8 +112,10 @@ def test_float(self, value, variable): [ (True, 'BOOL_TRUE_STRING_LIKE_INT'), (True, 'BOOL_TRUE_STRING_LIKE_BOOL'), + (True, 'BOOL_TRUE_STRING_LIKE_BOOL_WITH_COMMENT'), (True, 'BOOL_TRUE_INT'), (True, 'BOOL_TRUE_BOOL'), + (True, 'BOOL_TRUE_BOOL_WITH_COMMENT'), (True, 'BOOL_TRUE_STRING_1'), (True, 'BOOL_TRUE_STRING_2'), (True, 'BOOL_TRUE_STRING_3'), @@ -132,6 +134,10 @@ def test_bool_true(self, value, variable): def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' + def test_not_interpolated_proxied_value(self): + env = Env(interpolate=False) + assert env('PROXIED_VAR') == '$STR_VAR' + def test_escaped_dollar_sign(self): self.env.escape_proxy = True assert self.env('ESCAPED_VAR') == '$baz' @@ -175,9 +181,9 @@ def test_mix_tuple_issue_387(self): ) def test_str_list_with_spaces(self): - assert_type_and_value(list, [' foo', ' bar'], + assert_type_and_value(list, [' foo', ' spaces'], self.env('STR_LIST_WITH_SPACES', cast=[str])) - assert_type_and_value(list, [' foo', ' bar'], + assert_type_and_value(list, [' foo', ' spaces'], self.env.list('STR_LIST_WITH_SPACES')) def test_empty_list(self): @@ -339,6 +345,8 @@ def test_path(self): def test_smart_cast(self): assert self.env.get_value('STR_VAR', default='string') == 'bar' + assert self.env.get_value('STR_QUOTED_IGNORE_COMMENT', default='string') == 'foo' + assert self.env.get_value('STR_QUOTED_INCLUDE_HASH', default='string') == 'foo # with hash' assert self.env.get_value('BOOL_TRUE_STRING_LIKE_INT', default=True) assert not self.env.get_value( 'BOOL_FALSE_STRING_LIKE_INT', diff --git a/tests/test_env.txt b/tests/test_env.txt index 237489b9..d5480bf6 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -25,6 +25,8 @@ BOOL_TRUE_STRING_3='yes' BOOL_TRUE_STRING_4='y' BOOL_TRUE_STRING_5='true' BOOL_TRUE_BOOL=True +BOOL_TRUE_STRING_LIKE_BOOL_WITH_COMMENT='True' # comment +BOOL_TRUE_BOOL_WITH_COMMENT=True # comment BOOL_FALSE_STRING_LIKE_INT='0' BOOL_FALSE_INT=0 BOOL_FALSE_STRING_LIKE_BOOL='False' @@ -42,8 +44,11 @@ ESCAPED_VAR=\$baz EMPTY_LIST= EMPTY_INT_VAR= INT_VAR=42 -STR_LIST_WITH_SPACES= foo, bar +STR_LIST_WITH_SPACES= foo, spaces +STR_LIST_WITH_SPACES_QUOTED=' foo',' quoted' STR_VAR=bar +STR_QUOTED_IGNORE_COMMENT= 'foo' # comment +STR_QUOTED_INCLUDE_HASH='foo # with hash' # not comment MULTILINE_STR_VAR=foo\nbar MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---" MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END--- diff --git a/tests/test_expansion.py b/tests/test_expansion.py new file mode 100755 index 00000000..757ee9a2 --- /dev/null +++ b/tests/test_expansion.py @@ -0,0 +1,27 @@ +import pytest + +from environ import Env, Path +from environ.compat import ImproperlyConfigured + + +class TestExpansion: + def setup_method(self, method): + Env.ENVIRON = {} + self.env = Env() + self.env.read_env(Path(__file__, is_file=True)('test_expansion.txt')) + + def test_expansion(self): + assert self.env('HELLO') == 'Hello, world!' + + def test_braces(self): + assert self.env('BRACES') == 'Hello, world!' + + def test_recursion(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('RECURSIVE') + assert str(excinfo.value) == "Environment variable 'RECURSIVE' recursively references itself (eventually)" + + def test_transitive(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('R4') + assert str(excinfo.value) == "Environment variable 'R4' recursively references itself (eventually)" diff --git a/tests/test_expansion.txt b/tests/test_expansion.txt new file mode 100755 index 00000000..8290e45f --- /dev/null +++ b/tests/test_expansion.txt @@ -0,0 +1,9 @@ +VAR1='Hello' +VAR2='world' +HELLO="$VAR1, $VAR2!" +BRACES="${VAR1}, ${VAR2}!" +RECURSIVE="This variable is $RECURSIVE" +R1="$R2" +R2="$R3" +R3="$R4" +R4="$R1" diff --git a/tests/test_search.py b/tests/test_search.py index 0992bf98..a6d8f061 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -33,25 +33,45 @@ def test_solr_multicore_parsing(solr_url): @pytest.mark.parametrize( - 'url,engine', + 'url,engine,scheme', [ ('elasticsearch://127.0.0.1:9200/index', - 'elasticsearch_backend.ElasticsearchSearchEngine'), + 'elasticsearch_backend.ElasticsearchSearchEngine', + 'http',), + ('elasticsearchs://127.0.0.1:9200/index', + 'elasticsearch_backend.ElasticsearchSearchEngine', + 'https',), ('elasticsearch2://127.0.0.1:9200/index', - 'elasticsearch2_backend.Elasticsearch2SearchEngine'), + 'elasticsearch2_backend.Elasticsearch2SearchEngine', + 'http',), + ('elasticsearch2s://127.0.0.1:9200/index', + 'elasticsearch2_backend.Elasticsearch2SearchEngine', + 'https',), ('elasticsearch5://127.0.0.1:9200/index', - 'elasticsearch5_backend.Elasticsearch5SearchEngine'), + 'elasticsearch5_backend.Elasticsearch5SearchEngine', + 'http'), + ('elasticsearch5s://127.0.0.1:9200/index', + 'elasticsearch5_backend.Elasticsearch5SearchEngine', + 'https'), ('elasticsearch7://127.0.0.1:9200/index', - 'elasticsearch7_backend.Elasticsearch7SearchEngine'), + 'elasticsearch7_backend.Elasticsearch7SearchEngine', + 'http'), + ('elasticsearch7s://127.0.0.1:9200/index', + 'elasticsearch7_backend.Elasticsearch7SearchEngine', + 'https'), ], ids=[ 'elasticsearch', + 'elasticsearchs', 'elasticsearch2', + 'elasticsearch2s', 'elasticsearch5', + 'elasticsearch5s', 'elasticsearch7', + 'elasticsearch7s', ] ) -def test_elasticsearch_parsing(url, engine): +def test_elasticsearch_parsing(url, engine, scheme): """Ensure all supported Elasticsearch engines are recognized.""" timeout = 360 url = '{}?TIMEOUT={}'.format(url, timeout) @@ -63,6 +83,7 @@ def test_elasticsearch_parsing(url, engine): assert 'TIMEOUT' in url.keys() assert url['TIMEOUT'] == timeout assert 'PATH' not in url + assert url["URL"].startswith(scheme + ":") @pytest.mark.parametrize('storage', ['file', 'ram']) diff --git a/tox.ini b/tox.ini index 6251d4ab..d2367374 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -18,14 +18,13 @@ envlist = docs lint manifest - py{35,36,37,38,39,310,311}-django{111,22} + py{36,37,38,39,310,311}-django{111,22} py{36,37,38,39,310,311}-django{30,31,32} - py{38,39,310,311}-django{40,41} + py{38,39,310,311}-django{40,41,42} pypy-django{111,22,30,31,32} [gh-actions] python = - 3.5: py35 3.6: py36 3.7: py37 3.8: py38 @@ -45,6 +44,7 @@ deps = django32: Django>=3.2,<3.3 django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 + django42: Django>=4.2,<5.0 commands_pre = python -m pip install --upgrade pip python -m pip install . @@ -75,15 +75,12 @@ commands_pre = python -m pip install . commands = flake8 environ setup.py - # Format ("f") strings have not been introduced before Python 3.6, - # thus disable "consider-using-f-string" at this moment. pylint \ --logging-format-style=old \ --good-names-rgxs=m[0-9],f,v \ --disable=too-few-public-methods \ --disable=import-error \ --disable=unused-import \ - --disable=consider-using-f-string \ --disable=too-many-locals \ --disable=too-many-branches \ --disable=too-many-public-methods \