From e274fccba11cc75c7f165f4902070eec778c1b76 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:12:07 +0200 Subject: [PATCH 01/10] Fix pyOpenSSL patching routine The _lib is not accessible in more modern PyOpenSSL and thus won't need to be batched, better to not fail then. --- caramel/models.py | 6 +++++- caramel/scripts/generate_ca.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/caramel/models.py b/caramel/models.py index 92b3e4a..144ed17 100644 --- a/caramel/models.py +++ b/caramel/models.py @@ -30,7 +30,11 @@ def _crypto_patch(): https://github.com/pyca/pyopenssl/pull/115 has a pull&fix for it https://github.com/pyca/pyopenssl/issues/129 is an open issue about it.""" - _crypto._lib.ASN1_STRING_set_default_mask_asc(b"utf8only") + try: + _crypto._lib.ASN1_STRING_set_default_mask_asc(b"utf8only") + except AttributeError: + # Means we have a new OpenSSL and do not need to patch. + pass _crypto_patch() diff --git a/caramel/scripts/generate_ca.py b/caramel/scripts/generate_ca.py index 9623780..28af553 100644 --- a/caramel/scripts/generate_ca.py +++ b/caramel/scripts/generate_ca.py @@ -33,7 +33,11 @@ def _crypto_patch(): https://github.com/pyca/pyopenssl/pull/115 has a pull&fix for it https://github.com/pyca/pyopenssl/issues/129 is an open issue about it.""" - _crypto._lib.ASN1_STRING_set_default_mask_asc(b"utf8only") + try: + _crypto._lib.ASN1_STRING_set_default_mask_asc(b"utf8only") + except AttributeError: + # Means we have a new OpenSSL and do not need to patch. + pass _crypto_patch() From 5f8058e24375eba049fe8f002e85cc87a45a09f0 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:13:02 +0200 Subject: [PATCH 02/10] Make the models work with SqlAlchemy 2.0 in compat mode This uses the most basic update to the model code to work in SqlAlchemy 2.0. The declarative attribuetes may need an overhaul anyhow. --- caramel/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/caramel/models.py b/caramel/models.py index 144ed17..d2f1f08 100644 --- a/caramel/models.py +++ b/caramel/models.py @@ -94,6 +94,8 @@ def _fkcolumn(referent, *args, **kwargs): @as_declarative() class Base(object): + __allow_unmapped__ = True + @declared_attr # type: ignore def __tablename__(cls) -> str: # pylint: disable=no-self-argument return cls.__name__.lower() # pylint: disable=no-member @@ -209,14 +211,14 @@ def refreshable(cls): # Options subqueryload is to prevent thousands of small queries and # instead batch load the certificates at once - all_signed = _sa.select([Certificate.csr_id]) + all_signed = _sa.select(Certificate.csr_id) return ( cls.query().filter_by(rejected=False).filter(CSR.id.in_(all_signed)).all() ) @classmethod def unsigned(cls): - all_signed = _sa.select([Certificate.csr_id]) + all_signed = _sa.select(Certificate.csr_id) return ( cls.query() .filter_by(rejected=False) From d6f879f62fd0d6289d10d009ba28233e0cdea2a3 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:14:15 +0200 Subject: [PATCH 03/10] Consistently use UTC timestamp without TZ info This makes it explicit that we use UTC sans tzinfo, and thus avoid an warning from python 3.11+ datetime library. As the DB schema has no tzinfo, this makes it consistent. --- caramel/models.py | 8 +++++++- caramel/views.py | 4 ++-- tests/test_views.py | 7 ++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/caramel/models.py b/caramel/models.py index d2f1f08..62efb73 100644 --- a/caramel/models.py +++ b/caramel/models.py @@ -249,9 +249,15 @@ def __repr__(self): ).format(self) +def utcnow(): + """Return a non timezone-aware datetime in UTC.""" + ts = _datetime.datetime.now(_datetime.timezone.utc) + return ts.replace(tzinfo=None) + + class AccessLog(Base): # XXX: name could be better - when = _sa.Column(_sa.DateTime, default=_datetime.datetime.utcnow) + when = _sa.Column(_sa.DateTime, default=utcnow) # XXX: name could be better, could perhaps be limited length, # might not want this nullable addr = _sa.Column(_sa.Text) diff --git a/caramel/views.py b/caramel/views.py index ff42e99..14ae54c 100644 --- a/caramel/views.py +++ b/caramel/views.py @@ -1,7 +1,6 @@ #! /usr/bin/env python # vim: expandtab shiftwidth=4 softtabstop=4 tabstop=17 filetype=python : from hashlib import sha256 -from datetime import datetime from pyramid.response import Response from pyramid.view import view_config @@ -21,6 +20,7 @@ CSR, AccessLog, SigningCert, + utcnow, ) @@ -105,7 +105,7 @@ def cert_fetch(request): raise HTTPForbidden cert = csr.certificates.first() if cert: - if datetime.utcnow() < cert.not_after: + if utcnow() < cert.not_after: # XXX: appropriate content-type is ... ? return Response( cert.pem, content_type="application/octet-stream", charset="UTF-8" diff --git a/tests/test_views.py b/tests/test_views.py index 59188cb..761b55e 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -17,6 +17,7 @@ from caramel.models import ( CSR, AccessLog, + utcnow, ) from caramel import views @@ -137,7 +138,7 @@ def test_missing(self): def test_exists_valid(self): sha256sum = fixtures.CSRData.initial.sha256sum csr = CSR.by_sha256sum(sha256sum) - now = datetime.datetime.utcnow() + now = utcnow() self.req.matchdict["sha256"] = sha256sum resp = views.cert_fetch(self.req) # Verify response contents @@ -153,7 +154,7 @@ def test_exists_valid(self): def test_exists_expired(self): csr = fixtures.CSRData.with_expired_cert() csr.save() - now = datetime.datetime.utcnow() + now = utcnow() self.req.matchdict["sha256"] = csr.sha256sum resp = views.cert_fetch(self.req) # Verify response contents @@ -168,7 +169,7 @@ def test_exists_expired(self): def test_not_signed(self): csr = fixtures.CSRData.good() csr.save() - now = datetime.datetime.utcnow() + now = utcnow() self.req.matchdict["sha256"] = csr.sha256sum resp = views.cert_fetch(self.req) # Verify response contents From 7b10b60f2100f11c6db4ad80c8de96e7fd19b2b4 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:23:02 +0200 Subject: [PATCH 04/10] Fix generate req script because requests should have version=0 --- caramel/scripts/generate_ca.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/caramel/scripts/generate_ca.py b/caramel/scripts/generate_ca.py index 28af553..ab49a7b 100644 --- a/caramel/scripts/generate_ca.py +++ b/caramel/scripts/generate_ca.py @@ -12,6 +12,7 @@ ) from caramel import config +REQ_VERSION = 0x00 VERSION = 0x2 CA_BITS = 4096 # Subject attribs, in order. @@ -136,7 +137,7 @@ def create_ca_req(subject): key.generate_key(_crypto.TYPE_RSA, CA_BITS) req = _crypto.X509Req() - req.set_version(VERSION) + req.set_version(REQ_VERSION) req.set_pubkey(key) x509subject = req.get_subject() From 09aa6dc86d4fc05f8a653ef61651050dc8e2919a Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:27:15 +0200 Subject: [PATCH 05/10] Re-run modern black on the code --- caramel/config.py | 2 +- caramel/models.py | 7 ++----- caramel/scripts/generate_ca.py | 2 +- caramel/views.py | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/caramel/config.py b/caramel/config.py index f55b355..d1ef2fd 100644 --- a/caramel/config.py +++ b/caramel/config.py @@ -221,7 +221,7 @@ def get_log_level(argument_level, logger=None, env=None): logger = logging.getLogger() current_level = logger.level - argument_verbosity = logging.ERROR - argument_level * 10 # level steps are 10 + argument_verbosity = logging.ERROR - argument_level * 10 # level steps are 10 verbosity = min(argument_verbosity, env_level, current_level) log_level = ( verbosity if logging.DEBUG <= verbosity <= logging.ERROR else logging.ERROR diff --git a/caramel/models.py b/caramel/models.py index 62efb73..836d7e6 100644 --- a/caramel/models.py +++ b/caramel/models.py @@ -2,10 +2,7 @@ # vim: expandtab shiftwidth=4 softtabstop=4 tabstop=17 filetype=python : import sqlalchemy as _sa -from sqlalchemy.ext.declarative import ( - declared_attr, - as_declarative -) +from sqlalchemy.ext.declarative import declared_attr, as_declarative import sqlalchemy.orm as _orm from zope.sqlalchemy import register @@ -98,7 +95,7 @@ class Base(object): @declared_attr # type: ignore def __tablename__(cls) -> str: # pylint: disable=no-self-argument - return cls.__name__.lower() # pylint: disable=no-member + return cls.__name__.lower() # pylint: disable=no-member id = _sa.Column(_sa.Integer, primary_key=True) diff --git a/caramel/scripts/generate_ca.py b/caramel/scripts/generate_ca.py index ab49a7b..bf4d158 100644 --- a/caramel/scripts/generate_ca.py +++ b/caramel/scripts/generate_ca.py @@ -79,7 +79,7 @@ def later_check(subject): casubject = casubject[:-2] subject = subject[:-2] - for (ca, sub) in zip(casubject, subject): + for ca, sub in zip(casubject, subject): if ca != sub: raise ValueError("Subject needs to match CA cert:" "{}".format(casubject)) diff --git a/caramel/views.py b/caramel/views.py index 14ae54c..4670b05 100644 --- a/caramel/views.py +++ b/caramel/views.py @@ -28,7 +28,7 @@ # 2 kbyte should be enough for up to 4 kbit keys. # XXX: This should probably be handled outside of app (i.e. by the # server), or at least be configurable. -_MAXLEN = 2 * 2 ** 10 +_MAXLEN = 2 * 2**10 def raise_for_length(req, limit=_MAXLEN): @@ -38,7 +38,7 @@ def raise_for_length(req, limit=_MAXLEN): if length is None: raise HTTPLengthRequired if length > limit: - raise HTTPRequestEntityTooLarge("Max size: {0} kB".format(limit / 2 ** 10)) + raise HTTPRequestEntityTooLarge("Max size: {0} kB".format(limit / 2**10)) def raise_for_subject(components, required_prefix): From 694479c2ca7cea94a2b95c57a3618f19c6b69073 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:42:54 +0200 Subject: [PATCH 06/10] CI: allow workflow on manual trigger --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00b9e75..14c0c3b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ workflow: - if: $CI_EXTERNAL_PULL_REQUEST_IID - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "web" stages: - test From fffe8110c236cf3782cf127d5bba5ccd5e139ff3 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:52:21 +0200 Subject: [PATCH 07/10] CI: If machine-id doesn't exist, make it so --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14c0c3b..14733f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ caramel:systest: image: ${BUILD_IMAGE} before_script: - pip3 install . + - test -e /etc/machine-id || echo f26871a049f84136bb011f4744cde2dd > /etc/machine-id script: - make systest From 11d8bd1f70468561f85a6e1707688816844b5c12 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 17:55:01 +0200 Subject: [PATCH 08/10] CI: Ensure openssl is part of the systest container Because it helps to have it. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14733f8..40237b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,7 @@ caramel:systest: stage: test image: ${BUILD_IMAGE} before_script: + - dnf -y install openssl - pip3 install . - test -e /etc/machine-id || echo f26871a049f84136bb011f4744cde2dd > /etc/machine-id script: From d5bdc23a14019a54f851d464dd3ebfd76d4d8709 Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 18:07:46 +0200 Subject: [PATCH 09/10] Add a transient dep on legacy-cgi for python 3.13 deprecation --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index a28b598..e7d6b56 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,9 @@ "cryptography>=0.5.dev1", "pyOpenSSL>=0.14", "python-dateutil", + # Transient dependency from pyramid->webob, + # should be fixed in a later release of webob + "legacy-cgi; python_version >= '3.13'" ] deplinks = [] From 6b6e55e17f1180c4e92d7d6c91820193333ca02a Mon Sep 17 00:00:00 2001 From: "D.S. Ljungmark" Date: Thu, 18 Apr 2024 18:18:32 +0200 Subject: [PATCH 10/10] Allow failures for deprecation tests As of now, due to transient dependencies we cannot avoid deprecation warnings, and thus we should not make those be errors --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40237b6..acde084 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,12 @@ caramel:test: image: ${PYTHON_IMAGE} before_script: - pip3 install . + script: + - python3 -m unittest discover + +caramel:test:deprecation: + extends: caramel:test + allow_failure: true script: - python3 -W error::DeprecationWarning -m unittest discover