From ffd1d3a0c64e3f60623adf9ef30d358e742d7358 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 9 Aug 2022 12:43:37 -0400 Subject: [PATCH 1/2] fix: Cache producers so that they don't lose data (#21) Also needed to break one test into two so that caching did not interfere. --- CHANGELOG.rst | 10 +++++++++- edx_event_bus_kafka/__init__.py | 2 +- edx_event_bus_kafka/publishing/event_producer.py | 5 +++++ .../publishing/test_event_producer.py | 14 ++++++++++---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 31434e0c..a883efe7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,8 +16,16 @@ Unreleased * +[0.2.0] - 2022-08-09 +~~~~~~~~~~~~~~~~~~~~ + +Fixed +_____ + +* Cache producers so that they don't lose data. + [0.1.0] - 2022-06-16 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ Added _____ diff --git a/edx_event_bus_kafka/__init__.py b/edx_event_bus_kafka/__init__.py index 6ea8237e..63623c68 100644 --- a/edx_event_bus_kafka/__init__.py +++ b/edx_event_bus_kafka/__init__.py @@ -2,4 +2,4 @@ Kafka implementation for Open edX event bus. """ -__version__ = '0.1.1' +__version__ = '0.2.0' diff --git a/edx_event_bus_kafka/publishing/event_producer.py b/edx_event_bus_kafka/publishing/event_producer.py index 52c4c6db..aa891e38 100644 --- a/edx_event_bus_kafka/publishing/event_producer.py +++ b/edx_event_bus_kafka/publishing/event_producer.py @@ -115,6 +115,11 @@ def get_serializer(signal: OpenEdxPublicSignal) -> AvroSignalSerializer: return AvroSignalSerializer(signal) +# Note: This caching is required, since otherwise the Producer will +# fall out of scope and be garbage-collected, destroying the +# outbound-message queue and threads. The use of this cache allows the +# producers to be long-lived. +@lru_cache def get_producer_for_signal(signal: OpenEdxPublicSignal, event_key_field: str) -> Optional[SerializingProducer]: """ Create the producer for a signal and a key field path. diff --git a/edx_event_bus_kafka/publishing/test_event_producer.py b/edx_event_bus_kafka/publishing/test_event_producer.py index 1f5e4b1e..21259809 100644 --- a/edx_event_bus_kafka/publishing/test_event_producer.py +++ b/edx_event_bus_kafka/publishing/test_event_producer.py @@ -19,6 +19,11 @@ class TestEventProducer(TestCase): """Test producer.""" + def setUp(self): + super().setUp() + ep.get_producer_for_signal.cache_clear() + ep.get_serializer.cache_clear() + def test_extract_event_key(self): event_data = { 'user': UserData( @@ -56,17 +61,18 @@ def test_extract_key_schema(self): schema = ep.extract_key_schema(AvroSignalSerializer(signal), 'user.pii.username') assert schema == '{"name": "username", "type": "string"}' - def test_get_producer_for_signal(self): + def test_get_producer_for_signal_unconfigured(self): + """With missing essential settings, just warn and return None.""" signal = openedx_events.learning.signals.SESSION_LOGIN_COMPLETED - - # With missing essential settings, just warn and return None with warnings.catch_warnings(record=True) as caught_warnings: warnings.simplefilter('always') assert ep.get_producer_for_signal(signal, 'user.id') is None assert len(caught_warnings) == 1 assert str(caught_warnings[0].message).startswith("Cannot configure event-bus-kafka: Missing setting ") - # Creation succeeds when all settings are present + def test_get_producer_for_signal_configured(self): + """Creation succeeds when all settings are present.""" + signal = openedx_events.learning.signals.SESSION_LOGIN_COMPLETED with override_settings( SCHEMA_REGISTRY_URL='http://localhost:12345', SCHEMA_REGISTRY_API_KEY='some_key', From 52f0eb6ee118189278948930fc12d2312b4cd3d7 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 9 Aug 2022 13:39:32 -0400 Subject: [PATCH 2/2] docs: Fix badges and other cookiecutter-created repo/package references (#22) --- README.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index eda9670b..0a8740f3 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ One Time Setup .. code-block:: # Clone the repository - git clone git@github.com:edx/event-bus-kafka.git + git clone git@github.com:openedx/event-bus-kafka.git cd event-bus-kafka # Set up a virtualenv using virtualenvwrapper with the same name as the repo and activate it @@ -131,26 +131,26 @@ For more information about these options, see the `Getting Help`_ page. .. _community Slack workspace: https://openedx.slack.com/ .. _Getting Help: https://openedx.org/getting-help -.. |pypi-badge| image:: https://img.shields.io/pypi/v/event-bus-kafka.svg - :target: https://pypi.python.org/pypi/event-bus-kafka/ +.. |pypi-badge| image:: https://img.shields.io/pypi/v/edx-event-bus-kafka.svg + :target: https://pypi.python.org/pypi/edx-event-bus-kafka/ :alt: PyPI -.. |ci-badge| image:: https://github.com/edx/event-bus-kafka/workflows/Python%20CI/badge.svg?branch=main - :target: https://github.com/edx/event-bus-kafka/actions +.. |ci-badge| image:: https://github.com/openedx/event-bus-kafka/workflows/Python%20CI/badge.svg?branch=main + :target: https://github.com/openedx/event-bus-kafka/actions :alt: CI -.. |codecov-badge| image:: https://codecov.io/github/edx/event-bus-kafka/coverage.svg?branch=main - :target: https://codecov.io/github/edx/event-bus-kafka?branch=main +.. |codecov-badge| image:: https://codecov.io/github/openedx/event-bus-kafka/coverage.svg?branch=main + :target: https://codecov.io/github/openedx/event-bus-kafka?branch=main :alt: Codecov -.. |doc-badge| image:: https://readthedocs.org/projects/event-bus-kafka/badge/?version=latest - :target: https://event-bus-kafka.readthedocs.io/en/latest/ +.. |doc-badge| image:: https://readthedocs.org/projects/edx-event-bus-kafka/badge/?version=latest + :target: https://edx-event-bus-kafka.readthedocs.io/en/latest/ :alt: Documentation -.. |pyversions-badge| image:: https://img.shields.io/pypi/pyversions/event-bus-kafka.svg - :target: https://pypi.python.org/pypi/event-bus-kafka/ +.. |pyversions-badge| image:: https://img.shields.io/pypi/pyversions/edx-event-bus-kafka.svg + :target: https://pypi.python.org/pypi/edx-event-bus-kafka/ :alt: Supported Python versions -.. |license-badge| image:: https://img.shields.io/github/license/edx/event-bus-kafka.svg - :target: https://github.com/edx/event-bus-kafka/blob/main/LICENSE.txt +.. |license-badge| image:: https://img.shields.io/github/license/openedx/event-bus-kafka.svg + :target: https://github.com/openedx/event-bus-kafka/blob/main/LICENSE.txt :alt: License