Modernize python support (#307)

* chore: support Python >= 3.8

* chore: drop support for Python 2.7, 3.5, 3.6, 3.7.

* chore: remove Python 2.x compat layer

* Pin `pyramid<2.0dev` to defer 2.0 changes.

* fix: drop 'cryptacular' in favor of 'bcrypt' (Python 3.12 requirement)

* replace 'nosetests' with 'pytest'.

* tests: restore 100% coverage

* fix: remove use of deprecated 'pyramid.security.authenticated_userid' API

* fix: remove use of deprecated 'pyramid.session.check_csrf_token' API

* fix: remove use of deprecated 'pyramid.security.has_permission' API

* fix: remove use of deprecated 'pyramid.security.effective_principals' API

* fix: remove use of deprecated 'pyramid.session.UnencryptedCooki...' API

* chore: silence 'logging' module deprecation

* chore: silence deprecation spew from 'pkg_resources'

* chore: catch warning we are emitting about BBB test

* docs: use 'request.has_permission' in examples

* fix: remove invalid markup

* ci: add GHA workflow

.coveragerc

@@ -0,0 +1,8 @@
+[coverage:run]
+omit =
+    substanced/*/evolve.py
+    substanced/evolution/evolve*.py
+    substanced/scripts/*.py
+
+[coverage:report]
+show_missing = true

.github/workflows/main.yml

@@ -0,0 +1,66 @@
+name: Build and test
+
+on:
+    push:
+        branches:
+            - master
+        tags:
+    pull_request:
+
+jobs:
+    test:
+        strategy:
+            matrix:
+                py:
+                    - "3.8"
+                    - "3.9"
+                    - "3.10"
+                    - "3.11"
+                    - "3.12"
+                os:
+                    - "ubuntu-22.04"
+                architecture:
+                    - x64
+                include:
+                    # Only run coverage on ubuntu-22.04, except on pypy3
+                    - os: "ubuntu-22.04"
+                      pytest-args: "--cov"
+
+        name: "Python: ${{ matrix.py }}-${{ matrix.architecture }} on ${{ matrix.os }}"
+        runs-on: ${{ matrix.os }}
+        steps:
+            - uses: actions/checkout@v4
+            - name: Setup python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.py }}
+                  architecture: ${{ matrix.architecture }}
+            - run: pip install tox
+            - name: Running tox
+              run: tox -e py -- ${{ matrix.pytest-args }}
+
+    coverage:
+        runs-on: ubuntu-22.04
+        name: Validate coverage
+        steps:
+            - uses: actions/checkout@v4
+            - name: Setup python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: 3.12
+                  architecture: x64
+            - run: pip install tox
+            - run: tox -e cover
+
+    docs:
+        runs-on: ubuntu-22.04
+        name: Build the documentation
+        steps:
+            - uses: actions/checkout@v4
+            - name: Setup python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: 3.12
+                  architecture: x64
+            - run: pip install tox
+            - run: tox -e docs

CHANGES.txt

@@ -1,6 +1,16 @@
 1.0b1 (unreleased)
 ==================
 
+- Drop old Python versions (< 3.8), add newer ones (up to 3.12).
+  - Remove all the "straddle" compatibility stuff FBO Python 2.7
+  - Pin pyramid < 2.0dev, until it can be reviewed in depth.
+  - Fix deprecation warnings from pyramid.
+  - Silence deprecation warnings related to pkg_resources.
+  - Make ``py.test`` the testrunner (to support the newer Python versions).
+  - Replace ``cryptacular`` with plain ``bcrypt`` (the ``enscons`` builder
+    that ``cryptacular`` needs barfs on Python 3.12).
+  See: https://github.com/Pylons/substanced/pull/307
+
 - Override ``serialize`` for ``ReferenceIdSchemaNode``.
   See https://github.com/Pylons/substanced/pull/311
 

TODO.txt

@@ -109,7 +109,7 @@ Probably Bad Ideas
   how contained object can be acted upon?  E.g.::
 
     def __viewable__(self, context, request):
-        return has_permission('sdi.view', context, request)
+        return request.has_permission('sdi.view', context)
 
 
 Made Irrelevant

docs/locking.rst

@@ -17,9 +17,8 @@ To lock a resource:
 .. code-block:: python
 
    from substanced.locking import lock_resource
-   from pyramid.security import has_permission
 
-   if has_permission('sdi.lock', someresource, request):
+   if request.has_permission('sdi.lock', someresource):
        lock_resource(someresource, request.user, timeout=3600)
 
 If the resource is already locked by the owner supplied as ``owner_or_ownerid``
@@ -47,9 +46,8 @@ To unlock a resource:
 .. code-block:: python
 
    from substanced.locking import unlock_resource
-   from pyramid.security import has_permission
 
-   if has_permission('sdi.lock', someresource, request):
+   if request.has_permission('sdi.lock', someresource):
        unlock_resource(someresource, request.user)
 
 If the resource is already locked by a user other than the owner supplied as
@@ -73,9 +71,8 @@ To unlock a resource using an explicit lock token:
 .. code-block:: python
 
    from substanced.locking import unlock_token
-   from pyramid.security import has_permission
 
-   if has_permission('sdi.lock', someresource, request):
+   if request.has_permission('sdi.lock', someresource):
        unlock_token(someresource, token, request.user)
 
 If the lock identified by ``token`` belongs to a user other than the owner

pytest.ini

@@ -2,3 +2,6 @@
 addopts = -l --strict
 norecursedirs = lib include .tox .git
 python_files = test_*.py tests.py
+filterwarnings =
+     ignore::DeprecationWarning:pkg_resources
+     ignore::DeprecationWarning:zodburi

setup.cfg

@@ -1,13 +1,6 @@
 [easy_install]
 zip_ok = false
 
-[nosetests]
-match=^test
-where=substanced
-nocapture=1
-cover-package=substanced
-cover-erase=1
-
 [aliases]
 dev = develop easy_install substanced[testing]
 docs = develop easy_install substanced[docs]

setup.py

@@ -24,7 +24,7 @@
     README = CHANGES = ''
 
 install_requires = [
-    'pyramid>=1.5dev',  # route_name argument to resource_url
+    'pyramid>=1.5dev,<2.0dev',  # route_name argument to resource_url
     'ZODB',
     'hypatia>=0.2',  # query objects have intersection/union methods
     'venusian>=1.0a3',  # pyramid wants this too (prefer_finals...)
@@ -34,7 +34,7 @@
     'pyramid_zodbconn>=0.6',  # connection opened/closed events
     'pyramid_chameleon',
     'pyramid_mailer',
-    'cryptacular',
+    'bcrypt',
     'python-magic',
     'PyYAML',
     'zope.copy',
@@ -70,13 +70,12 @@
       classifiers=[
         "Intended Audience :: Developers",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Framework :: Pyramid",
         "Topic :: Internet :: WWW/HTTP :: WSGI",

substanced/audit/__init__.py

@@ -4,7 +4,7 @@
 from persistent import Persistent
 from ZODB.POSException import ConflictError
 
-from pyramid.compat import is_nonstr_iter
+from ..util import is_nonstr_iter
 
 class LayerFull(Exception):
     pass

substanced/audit/views.py

@@ -3,7 +3,6 @@
 
 from pyramid.view import view_defaults
 from pyramid.httpexceptions import HTTPPreconditionFailed
-from pyramid.compat import PY3, text_type
 from substanced.sdi import mgmt_view, RIGHT
 from substanced.util import (
     Batch,
@@ -99,7 +98,7 @@ def auditstream_sse(self):
                 oids = [get_oid(self.context)]
             _gen, _idx = map(int, last_event_id.split('-', 1))
             events = log.newer(_gen, _idx, oids=oids)
-            msg = text_type('')
+            msg = ''
             for gen, idx, event in events:
                 event_id = '%s-%s' % (gen, idx)
                 message = compose_message(event_id, event.name, event.payload)
@@ -118,8 +117,4 @@ def compose_message(eventid, name=None, payload=''):
     msg += 'retry: 10000\n'
     msg += 'data: %s\n' % payload
     msg += '\n'
-    if PY3: # pragma: no cover
-        return msg
-    else: # pragma: no cover
-        return msg.decode('utf-8')
-
+    return msg

substanced/catalog/__init__.py

@@ -32,8 +32,6 @@
 from ..objectmap import find_objectmap
 from ..stats import statsd_timer
 from ..util import get_oid
-from .._compat import INT_TYPES
-from .._compat import u
 
 from .factories import (
     IndexFactory,
@@ -70,7 +68,7 @@
 
 logger = logging.getLogger(__name__) # API
 
-_SLASH = u('/')
+_SLASH = '/'
 
 _marker = object()
 
@@ -174,7 +172,7 @@ def unindex_resource(self, resource_or_oid, action_mode=None):
         which explicitly indicates that you'd like to use the index's
         action_mode value."""
         oid = get_oid(resource_or_oid, resource_or_oid)
-        if not isinstance(oid, INT_TYPES):
+        if not isinstance(oid, int):
             raise ValueError(
                 'resource_or_oid must be a resource object with an __oid__ '
                 'attribute or an integer oid'

substanced/catalog/indexes.py

@@ -1,3 +1,5 @@
+from urllib.parse import unquote as url_unquote
+
 import colander
 import deform.widget
 import re
@@ -11,12 +13,7 @@
 import hypatia.text
 import hypatia.util
 from persistent import Persistent
-from pyramid.compat import (
-    url_unquote_text,
-    is_nonstr_iter,
-    )
 from pyramid.settings import asbool
-from pyramid.security import effective_principals
 from pyramid.traversal import resource_path_tuple
 from pyramid.interfaces import IRequest
 from zope.interface import implementer
@@ -31,18 +28,16 @@
 from ..property import PropertySheet
 from ..schema import Schema
 from ..stats import statsd_timer
-from .._compat import STRING_TYPES
-from .._compat import INT_TYPES
-from .._compat import u
+from ..util import is_nonstr_iter
 
 from .discriminators import dummy_discriminator
 from .util import oid_from_resource
 
 from . import deferred
 
 PATH_WITH_OPTIONS = re.compile(r'\[(.+?)\](.+?)$')
-_BLANK = u('')
-_SLASH = u('/')
+_BLANK = ''
+_SLASH = '/'
 
 _marker = object()
 
@@ -107,7 +102,7 @@ def reindex_resource(self, resource, oid=None, action_mode=None):
             self.add_action(action)
 
     def unindex_resource(self, resource_or_oid, action_mode=None):
-        if isinstance(resource_or_oid, INT_TYPES):
+        if isinstance(resource_or_oid, int):
             oid = resource_or_oid
         else:
             oid = oid_from_resource(resource_or_oid)
@@ -243,7 +238,7 @@ def _parse_path_str(self, path_str):
         if not path.startswith('/'):
             raise ValueError('Path must start with a slash')
 
-        tmp = [x for x in url_unquote_text(path).split(_SLASH) if x]
+        tmp = [x for x in url_unquote(path).split(_SLASH) if x]
         path_tuple = (_BLANK,) + tuple(tmp)
         return path_tuple, depth, include_origin
 
@@ -253,7 +248,7 @@ def _parse_path(self, obj_or_path):
         path_tuple = obj_or_path
         if hasattr(obj_or_path, '__parent__'):
             path_tuple = resource_path_tuple(obj_or_path)
-        elif isinstance(obj_or_path, STRING_TYPES):
+        elif isinstance(obj_or_path, str):
             path_tuple, depth, include_origin = self._parse_path_str(
                 obj_or_path)
         elif not isinstance(obj_or_path, tuple):
@@ -422,7 +417,7 @@ def allows(self, principals, permission):
         ``permission`` must be a permission name.
         """
         if IRequest.providedBy(principals):
-            principals = effective_principals(principals)
+            principals = principals.effective_principals
         elif not is_nonstr_iter(principals):
             principals = (principals,)
         return AllowsComparator(self, (principals, permission))

substanced/catalog/subscribers.py

@@ -144,7 +144,7 @@ def on_startup(event):
             # evolve because autosync is on and causes errors, cant autosync
             # because evolve steps havent been run), so we avoid doing any
             # sync if there are unfinished evolve steps
-            logger.warn(
+            logger.warning(
                 'Cannot autosync/autoreindex catalog due to unfinished evolve '
                 'steps'
                 )